From 2dbe7904bea6e258a90f19a8324a6de4de6ce6b3 Mon Sep 17 00:00:00 2001 From: "Su, Xiaomeng" Date: Fri, 1 Mar 2024 10:53:24 +0000 Subject: [PATCH] dli_umn_20240227_version Reviewed-by: Pruthi, Vineet Co-authored-by: Su, Xiaomeng Co-committed-by: Su, Xiaomeng --- docs/dli/umn/ALL_META.TXT.json | 105 +- docs/dli/umn/CLASS.TXT.json | 42 +- docs/dli/umn/dli_01_00006.html | 16 +- docs/dli/umn/dli_01_0003.html | 7 +- docs/dli/umn/dli_01_0005.html | 12 +- docs/dli/umn/dli_01_0012.html | 2 +- docs/dli/umn/dli_01_0013.html | 1 - docs/dli/umn/dli_01_0017.html | 9 +- docs/dli/umn/dli_01_0318.html | 4 +- docs/dli/umn/dli_01_0320.html | 55 +- docs/dli/umn/dli_01_0363.html | 34 +- docs/dli/umn/dli_01_0378.html | 11 +- docs/dli/umn/dli_01_0384.html | 32 +- docs/dli/umn/dli_01_0385.html | 6 +- docs/dli/umn/dli_01_0397.html | 2952 ++++++++++++++++- docs/dli/umn/dli_01_0402.html | 2 +- docs/dli/umn/dli_01_0403.html | 45 +- docs/dli/umn/dli_01_0410.html | 2 +- docs/dli/umn/dli_01_0421.html | 5 +- docs/dli/umn/dli_01_0422.html | 2 +- docs/dli/umn/dli_01_0440.html | 73 +- docs/dli/umn/dli_01_0441.html | 2 +- docs/dli/umn/dli_01_0445.html | 292 +- docs/dli/umn/dli_01_0447.html | 2 +- docs/dli/umn/dli_01_0448.html | 12 +- docs/dli/umn/dli_01_0451.html | 2 +- docs/dli/umn/dli_01_0455.html | 2 +- docs/dli/umn/dli_01_0457.html | 2 +- docs/dli/umn/dli_01_0476.html | 12 +- docs/dli/umn/dli_01_0487.html | 2 +- docs/dli/umn/dli_01_0498.html | 6 +- docs/dli/umn/dli_01_0512.html | 8 +- docs/dli/umn/dli_01_0531.html | 3 +- docs/dli/umn/dli_01_0534.html | 33 +- docs/dli/umn/dli_01_0538.html | 2 +- docs/dli/umn/dli_01_0550.html | 2 +- docs/dli/umn/dli_01_0552.html | 2 +- docs/dli/umn/dli_01_0559.html | 25 +- docs/dli/umn/dli_01_0561.html | 8 +- docs/dli/umn/dli_01_0567.html | 2 +- docs/dli/umn/dli_03_0017.html | 6 +- docs/dli/umn/dli_03_0032.html | 2 +- docs/dli/umn/dli_03_0037.html | 2 +- docs/dli/umn/dli_03_0044.html | 62 +- docs/dli/umn/dli_03_0045.html | 2 +- docs/dli/umn/dli_03_0056.html | 2 +- docs/dli/umn/dli_03_0071.html | 4 +- docs/dli/umn/dli_03_0072.html | 2 +- docs/dli/umn/dli_03_0082.html | 2 +- docs/dli/umn/dli_03_0100.html | 56 +- docs/dli/umn/dli_03_0125.html | 4 +- docs/dli/umn/dli_03_0136.html | 59 +- docs/dli/umn/dli_03_0139.html | 2 +- docs/dli/umn/dli_03_0140.html | 2 +- docs/dli/umn/dli_03_0161.html | 2 +- docs/dli/umn/dli_03_0176.html | 2 +- docs/dli/umn/dli_03_0208.html | 2 +- docs/dli/umn/dli_03_0209.html | 2 +- docs/dli/umn/dli_03_0210.html | 4 +- docs/dli/umn/dli_03_0227.html | 2 +- docs/dli/umn/dli_03_0264.html | 14 +- docs/dli/umn/dli_07_0005.html | 18 +- docs/dli/umn/dli_07_0006.html | 45 +- docs/dli/umn/dli_07_0009.html | 2 +- docs/dli/umn/en-us_image_0000001209516352.png | Bin 632 -> 0 bytes docs/dli/umn/en-us_image_0000001478148728.png | Bin 632 -> 0 bytes docs/dli/umn/en-us_image_0000001668054032.png | Bin 0 -> 124080 bytes docs/dli/umn/en-us_image_0000001716012053.png | Bin 0 -> 180498 bytes docs/dli/umn/en-us_image_0000001835430521.png | Bin 0 -> 957 bytes 69 files changed, 3634 insertions(+), 502 deletions(-) delete mode 100644 docs/dli/umn/en-us_image_0000001209516352.png delete mode 100644 docs/dli/umn/en-us_image_0000001478148728.png create mode 100644 docs/dli/umn/en-us_image_0000001668054032.png create mode 100644 docs/dli/umn/en-us_image_0000001716012053.png create mode 100644 docs/dli/umn/en-us_image_0000001835430521.png diff --git a/docs/dli/umn/ALL_META.TXT.json b/docs/dli/umn/ALL_META.TXT.json index 8566a8655..2b5b0b17a 100644 --- a/docs/dli/umn/ALL_META.TXT.json +++ b/docs/dli/umn/ALL_META.TXT.json @@ -1,4 +1,7 @@ [ + { + "dockw":"User Guide" + }, { "uri":"dli_01_0538.html", "node_id":"dli_01_0538.xml", @@ -11,10 +14,7 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", - "IsMulti":"No", - "IsBot":"Yes" + "documenttype":"usermanual" } ], "title":"Service Overview", @@ -52,9 +52,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -89,7 +88,7 @@ "code":"5", "des":"Only the latest 100 jobs are displayed on DLI's SparkUI.A maximum of 1,000 job results can be displayed on the console. To view more or all jobs, export the job data to O", "doc_type":"usermanual", - "kw":"Constraints,Service Overview,User Guide", + "kw":"Constraints and Limitations,Service Overview,User Guide", "search_title":"", "metedata":[ { @@ -99,7 +98,7 @@ "IsBot":"Yes" } ], - "title":"Constraints", + "title":"Constraints and Limitations", "githuburl":"" }, { @@ -134,9 +133,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -216,9 +214,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -237,9 +234,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -331,15 +327,16 @@ { "uri":"dli_01_0567.html", "node_id":"dli_01_0567.xml", - "product_code":"", + "product_code":"dli", "code":"17", "des":"DLI provides the following job types:SQL job: SQL jobs provide you with standard SQL statements and are compatible with Spark SQL and Presto SQL (based on Presto). You ca", - "doc_type":"", + "doc_type":"usermanual", "kw":"Overview,Job Management,User Guide", "search_title":"", "metedata":[ { - + "prodname":"dli", + "documenttype":"usermanual" } ], "title":"Overview", @@ -621,9 +618,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -799,7 +795,7 @@ "node_id":"dli_01_0487.xml", "product_code":"dli", "code":"40", - "des":"Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.If Status of queue xxx is assigning, which is not available is ", + "des":"Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.Queues with 16 CUs do not support scale-out or scale-in.Queues ", "doc_type":"usermanual", "kw":"Elastic Queue Scaling,Queue Management,User Guide", "search_title":"", @@ -860,9 +856,9 @@ "node_id":"dli_01_0421.xml", "product_code":"dli", "code":"43", - "des":"Once you have created a message notification topic, you can Add subscription of the topic on the Topic Management page of the Simple Message Notification service. You can", + "des":"Once you have created an SMN topic, you can easily subscribe to it by going to the Topic Management > Topics page of the SMN console. You can choose to receive notificati", "doc_type":"usermanual", - "kw":"Creating a Message Notification Topic,Queue Management,User Guide", + "kw":"Creating an SMN Topic,Queue Management,User Guide", "search_title":"", "metedata":[ { @@ -872,7 +868,7 @@ "IsBot":"Yes" } ], - "title":"Creating a Message Notification Topic", + "title":"Creating an SMN Topic", "githuburl":"" }, { @@ -887,9 +883,7 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" + "documenttype":"usermanual" } ], "title":"Managing Queue Tags", @@ -962,16 +956,15 @@ "node_id":"dli_01_0447.xml", "product_code":"dli", "code":"48", - "des":"You can isolate databases allocated to different users by setting permissions to ensure data query performance.The administrator and database owner have all permissions, ", + "des":"By setting permissions, you can assign varying database permissions to different users.The administrator and database owner have all permissions, which cannot be set or m", "doc_type":"usermanual", "kw":"Managing Database Permissions,Databases and Tables,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -983,16 +976,15 @@ "node_id":"dli_01_0448.xml", "product_code":"dli", "code":"49", - "des":"You can isolate databases allocated to different users by setting permissions to ensure data query performance.The administrator and database owner have all permissions, ", + "des":"By setting permissions, you can assign varying table permissions to different users.The administrator and table owner have all permissions, which cannot be set or modifie", "doc_type":"usermanual", "kw":"Managing Table Permissions,Databases and Tables,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -1300,9 +1292,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -1459,7 +1450,7 @@ "node_id":"dli_01_0003.xml", "product_code":"dli", "code":"72", - "des":"Typically, you cannot use DLI to directly access a data source in a VPC other than the one where DLI is because the network between DLI and the data source is disconnecte", + "des":"In cross-source data analysis scenarios, DLI needs to connect to external data sources. However, due to the different VPCs between the data source and DLI, the network ca", "doc_type":"usermanual", "kw":"Overview,Enhanced Datasource Connections,User Guide", "search_title":"", @@ -1690,9 +1681,9 @@ "node_id":"dli_01_0561.xml", "product_code":"dli", "code":"83", - "des":"Datasource authentication is used to manage authentication information for accessing specified data sources. After datasource authentication is configured, you do not nee", + "des":"When analyzing across multiple sources, it is not recommended to configure authentication information directly in a job as it can lead to password leakage. Instead, you a", "doc_type":"usermanual", - "kw":"Introduction,Datasource Authentication,User Guide", + "kw":"Overview,Datasource Authentication,User Guide", "search_title":"", "metedata":[ { @@ -1703,7 +1694,7 @@ "IsBot":"Yes" } ], - "title":"Introduction", + "title":"Overview", "githuburl":"" }, { @@ -1905,10 +1896,7 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", - "IsMulti":"No", - "IsBot":"Yes" + "documenttype":"usermanual" } ], "title":"Permissions Management", @@ -1926,9 +1914,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -2091,9 +2078,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -2112,9 +2098,8 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No", + "documenttype":"usermanual", "IsBot":"Yes" } ], @@ -2840,9 +2825,9 @@ "node_id":"dli_03_0136.xml", "product_code":"dli", "code":"138", - "des":"Check the following operations:Adjusting or adding optimization parameters or the number of concurrent threads of a job, or modifying Flink SQL statements or a Flink Jar ", + "des":"Flink's checkpointing is a fault tolerance and recovery mechanism. This mechanism ensures that real-time programs can self-recover in case of exceptions or machine issues", "doc_type":"usermanual", - "kw":"How Do I Know Whether a Flink Job Can Be Restored from a Checkpoint After Being Restarted?,Flink Job", + "kw":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?,Flink Jobs,Use", "search_title":"", "metedata":[ { @@ -2853,7 +2838,7 @@ "IsBot":"Yes" } ], - "title":"How Do I Know Whether a Flink Job Can Be Restored from a Checkpoint After Being Restarted?", + "title":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?", "githuburl":"" }, { @@ -3134,7 +3119,7 @@ "node_id":"dli_03_0072.xml", "product_code":"dli", "code":"152", - "des":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket. For details, s", + "des":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket.Use the OBS cro", "doc_type":"usermanual", "kw":"How Do I Synchronize DLI Table Data from One Region to Another?,Problems Related to SQL Jobs,User Gu", "search_title":"", @@ -3218,7 +3203,7 @@ "node_id":"dli_03_0071.xml", "product_code":"dli", "code":"156", - "des":"Check whether the account is in arrears. If yes, recharge the account.If the error persists, log out and log in again.", + "des":"Check if your account is in arrears and top it up if necessary.If the same error message persists after the top-up, log out of your account and log back in.", "doc_type":"usermanual", "kw":"Why Is Error \"The current account does not have permission to perform this operation,the current acc", "search_title":"", @@ -3386,7 +3371,7 @@ "node_id":"dli_03_0208.xml", "product_code":"dli", "code":"164", - "des":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.View the OBS bucket in the AccessControlException and check whether you are us", + "des":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.Check the OBS bucket written in the AccessControlException to confirm if your ", "doc_type":"usermanual", "kw":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?,Problems Relat", "search_title":"", @@ -3407,7 +3392,7 @@ "node_id":"dli_03_0209.xml", "product_code":"dli", "code":"165", - "des":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Th", + "des":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Yo", "doc_type":"usermanual", "kw":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{b", "search_title":"", @@ -3890,7 +3875,7 @@ "node_id":"dli_03_0017.xml", "product_code":"dli", "code":"188", - "des":"If the AK and SK are obtained, set the parameters as follows:Create SparkContext using codeval sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.ac", + "des":"Hard-coded or plaintext AK and SK pose significant security risks. To ensure security, encrypt your AK and SK, store them in configuration files or environment variables,", "doc_type":"usermanual", "kw":"How Do I Set the AK/SK for a Queue to Operate an OBS Table?,Problems Related to Spark Jobs,User Guid", "search_title":"", @@ -4352,7 +4337,7 @@ "node_id":"dli_03_0264.xml", "product_code":"dli", "code":"210", - "des":"Log in to the management console.Click in the upper left corner and select Region and Project.Click (the My Quotas icon) in the upper right corner.The Service Quota pag", + "des":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.The Serv", "doc_type":"usermanual", "kw":"Why Is Error \"Failed to create the database. {\"error_code\":\"DLI.1028\";\"error_msg\":\"Already reached t", "search_title":"", @@ -5423,9 +5408,9 @@ "node_id":"dli_03_0125.xml", "product_code":"dli", "code":"261", - "des":"If different IAM accounts call APIs in the same enterprise project in the same region, the accounts use the same project ID.", + "des":"When different IAM users call an API under the same enterprise project in the same region, the project ID is the same.", "doc_type":"usermanual", - "kw":"Are Project IDs of Different Accounts the Same When They Are Used to Call APIs?,APIs,User Guide", + "kw":"Is the Project ID Fixed when Different IAM Users Call an API?,APIs,User Guide", "search_title":"", "metedata":[ { @@ -5436,7 +5421,7 @@ "IsBot":"Yes" } ], - "title":"Are Project IDs of Different Accounts the Same When They Are Used to Call APIs?", + "title":"Is the Project ID Fixed when Different IAM Users Call an API?", "githuburl":"" }, { diff --git a/docs/dli/umn/CLASS.TXT.json b/docs/dli/umn/CLASS.TXT.json index 829ad1978..fb44d47c7 100644 --- a/docs/dli/umn/CLASS.TXT.json +++ b/docs/dli/umn/CLASS.TXT.json @@ -38,7 +38,7 @@ { "desc":"Only the latest 100 jobs are displayed on DLI's SparkUI.A maximum of 1,000 job results can be displayed on the console. To view more or all jobs, export the job data to O", "product_code":"dli", - "title":"Constraints", + "title":"Constraints and Limitations", "uri":"dli_07_0005.html", "doc_type":"usermanual", "p_code":"1", @@ -145,10 +145,10 @@ }, { "desc":"DLI provides the following job types:SQL job: SQL jobs provide you with standard SQL statements and are compatible with Spark SQL and Presto SQL (based on Presto). You ca", - "product_code":"", + "product_code":"dli", "title":"Overview", "uri":"dli_01_0567.html", - "doc_type":"", + "doc_type":"usermanual", "p_code":"16", "code":"17" }, @@ -351,7 +351,7 @@ "code":"39" }, { - "desc":"Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.If Status of queue xxx is assigning, which is not available is ", + "desc":"Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.Queues with 16 CUs do not support scale-out or scale-in.Queues ", "product_code":"dli", "title":"Elastic Queue Scaling", "uri":"dli_01_0487.html", @@ -378,9 +378,9 @@ "code":"42" }, { - "desc":"Once you have created a message notification topic, you can Add subscription of the topic on the Topic Management page of the Simple Message Notification service. You can", + "desc":"Once you have created an SMN topic, you can easily subscribe to it by going to the Topic Management > Topics page of the SMN console. You can choose to receive notificati", "product_code":"dli", - "title":"Creating a Message Notification Topic", + "title":"Creating an SMN Topic", "uri":"dli_01_0421.html", "doc_type":"usermanual", "p_code":"34", @@ -423,7 +423,7 @@ "code":"47" }, { - "desc":"You can isolate databases allocated to different users by setting permissions to ensure data query performance.The administrator and database owner have all permissions, ", + "desc":"By setting permissions, you can assign varying database permissions to different users.The administrator and database owner have all permissions, which cannot be set or m", "product_code":"dli", "title":"Managing Database Permissions", "uri":"dli_01_0447.html", @@ -432,7 +432,7 @@ "code":"48" }, { - "desc":"You can isolate databases allocated to different users by setting permissions to ensure data query performance.The administrator and database owner have all permissions, ", + "desc":"By setting permissions, you can assign varying table permissions to different users.The administrator and table owner have all permissions, which cannot be set or modifie", "product_code":"dli", "title":"Managing Table Permissions", "uri":"dli_01_0448.html", @@ -639,7 +639,7 @@ "code":"71" }, { - "desc":"Typically, you cannot use DLI to directly access a data source in a VPC other than the one where DLI is because the network between DLI and the data source is disconnecte", + "desc":"In cross-source data analysis scenarios, DLI needs to connect to external data sources. However, due to the different VPCs between the data source and DLI, the network ca", "product_code":"dli", "title":"Overview", "uri":"dli_01_0003.html", @@ -738,9 +738,9 @@ "code":"82" }, { - "desc":"Datasource authentication is used to manage authentication information for accessing specified data sources. After datasource authentication is configured, you do not nee", + "desc":"When analyzing across multiple sources, it is not recommended to configure authentication information directly in a job as it can lead to password leakage. Instead, you a", "product_code":"dli", - "title":"Introduction", + "title":"Overview", "uri":"dli_01_0561.html", "doc_type":"usermanual", "p_code":"82", @@ -1233,9 +1233,9 @@ "code":"137" }, { - "desc":"Check the following operations:Adjusting or adding optimization parameters or the number of concurrent threads of a job, or modifying Flink SQL statements or a Flink Jar ", + "desc":"Flink's checkpointing is a fault tolerance and recovery mechanism. This mechanism ensures that real-time programs can self-recover in case of exceptions or machine issues", "product_code":"dli", - "title":"How Do I Know Whether a Flink Job Can Be Restored from a Checkpoint After Being Restarted?", + "title":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?", "uri":"dli_03_0136.html", "doc_type":"usermanual", "p_code":"106", @@ -1359,7 +1359,7 @@ "code":"151" }, { - "desc":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket. For details, s", + "desc":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket.Use the OBS cro", "product_code":"dli", "title":"How Do I Synchronize DLI Table Data from One Region to Another?", "uri":"dli_03_0072.html", @@ -1395,7 +1395,7 @@ "code":"155" }, { - "desc":"Check whether the account is in arrears. If yes, recharge the account.If the error persists, log out and log in again.", + "desc":"Check if your account is in arrears and top it up if necessary.If the same error message persists after the top-up, log out of your account and log back in.", "product_code":"dli", "title":"Why Is Error \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget.\" Reported when a SQL Statement Is Executed?", "uri":"dli_03_0071.html", @@ -1467,7 +1467,7 @@ "code":"163" }, { - "desc":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.View the OBS bucket in the AccessControlException and check whether you are us", + "desc":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.Check the OBS bucket written in the AccessControlException to confirm if your ", "product_code":"dli", "title":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?", "uri":"dli_03_0208.html", @@ -1476,7 +1476,7 @@ "code":"164" }, { - "desc":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Th", + "desc":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Yo", "product_code":"dli", "title":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" Reported When I Access a SQL Job?", "uri":"dli_03_0209.html", @@ -1683,7 +1683,7 @@ "code":"187" }, { - "desc":"If the AK and SK are obtained, set the parameters as follows:Create SparkContext using codeval sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.ac", + "desc":"Hard-coded or plaintext AK and SK pose significant security risks. To ensure security, encrypt your AK and SK, store them in configuration files or environment variables,", "product_code":"dli", "title":"How Do I Set the AK/SK for a Queue to Operate an OBS Table?", "uri":"dli_03_0017.html", @@ -1881,7 +1881,7 @@ "code":"209" }, { - "desc":"Log in to the management console.Click in the upper left corner and select Region and Project.Click (the My Quotas icon) in the upper right corner.The Service Quota pag", + "desc":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.The Serv", "product_code":"dli", "title":"Why Is Error \"Failed to create the database. {\"error_code\":\"DLI.1028\";\"error_msg\":\"Already reached the maximum quota of databases:XXX\".\" Reported?", "uri":"dli_03_0264.html", @@ -2340,9 +2340,9 @@ "code":"260" }, { - "desc":"If different IAM accounts call APIs in the same enterprise project in the same region, the accounts use the same project ID.", + "desc":"When different IAM users call an API under the same enterprise project in the same region, the project ID is the same.", "product_code":"dli", - "title":"Are Project IDs of Different Accounts the Same When They Are Used to Call APIs?", + "title":"Is the Project ID Fixed when Different IAM Users Call an API?", "uri":"dli_03_0125.html", "doc_type":"usermanual", "p_code":"259", diff --git a/docs/dli/umn/dli_01_00006.html b/docs/dli/umn/dli_01_00006.html index 4248eb354..e5b29e84f 100644 --- a/docs/dli/umn/dli_01_00006.html +++ b/docs/dli/umn/dli_01_00006.html @@ -8,7 +8,21 @@ -

2023-10-08

+

2024-02-27

+ +

Added the following content:

+ +

Modified the following section:

+

Changed DDS MongoDB to DDS in Creating a Flink Jar Job.

+ + +

2023-11-01

+ +

Modified the following content:

+ + + +

2023-10-08

Modified the following content:

diff --git a/docs/dli/umn/dli_01_0003.html b/docs/dli/umn/dli_01_0003.html index 209cf664a..fe5fa05cf 100644 --- a/docs/dli/umn/dli_01_0003.html +++ b/docs/dli/umn/dli_01_0003.html @@ -1,9 +1,12 @@

Overview

-

What Is Enhanced Datasource Connection?

Typically, you cannot use DLI to directly access a data source in a VPC other than the one where DLI is because the network between DLI and the data source is disconnected. For proper access, you need to establish a network connection between them.

-

DLI provides enhanced connections. Establishing a VPC peering connection allows DLI to communicate with the VPC of the data source, supporting cross-source data analysis.

+

What Is Enhanced Datasource Connection?

In cross-source data analysis scenarios, DLI needs to connect to external data sources. However, due to the different VPCs between the data source and DLI, the network cannot be connected, which results in DLI being unable to read data from the data source. DLI's enhanced datasource connection feature enables network connectivity between DLI and the data source.

+

This section will introduce a solution for cross-VPC data source network connectivity:

+
  • Creating an enhanced datasource connection: Establish a VPC peering connection to connect DLI and the data source's VPC network.
  • Testing network connectivity: Verify the connectivity between the queue and the data source's network.

For details about the data sources that support cross-source access, see Cross-Source Analysis Development Methods.

+

In cross-source development scenarios, there is a risk of password leakage if datasource authentication information is directly configured. You are advised to use the datasource authentication provided by DLI. For details, see Overview.

+

Constraints

  • Datasource connections cannot be created for the default queue.
  • Flink jobs can directly access DIS, OBS, and SMN data sources without using datasource connections.
  • VPC Administrator permissions are required for enhanced connections to use VPCs, subnets, routes, VPC peering connections.
  • If you use an enhanced datasource connection, the CIDR block of the elastic resource pool or queue cannot overlap with that of the data source.
  • Only queues bound with datasource connections can access datasource tables.
  • Datasource tables do not support the preview function.
  • When checking the connectivity of datasource connections, the constraints on IP addresses are as follows:
    • The IP address must be valid, which consists of four decimal numbers separated by periods (.). The value ranges from 0 to 255.
    • During the test, you can add a port after the IP address and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

      For example, 192.168.xx.xx or 192.168.xx.xx:8181.

    diff --git a/docs/dli/umn/dli_01_0005.html b/docs/dli/umn/dli_01_0005.html index 9460e314b..f325c7106 100644 --- a/docs/dli/umn/dli_01_0005.html +++ b/docs/dli/umn/dli_01_0005.html @@ -52,9 +52,9 @@

Creating a Table

Before creating a table, ensure that a database has been created.

-
  1. You can create a table on either the Databases and Tables page or the SQL Editor page.

    Datasource connection tables, such as View tables, HBase (MRS) tables, OpenTSDB (MRS) tables, DWS tables, RDS tables, and CSS tables, cannot be created. You can use SQL to create views and datasource connection tables. For details, see sections Creating a View and Creating a Datasource Connection Table in the Data Lake Insight SQL Syntax Reference.

    +
    1. You can create a table on either the Databases and Tables page or the SQL Editor page.

      Datasource connection tables, such as View tables, HBase (MRS) tables, OpenTSDB (MRS) tables, DWS tables, RDS tables, and CSS tables, cannot be created. You can use SQL to create views and datasource connection tables. For details, see sections Creating a View and Creating a Datasource Connection Table in the Data Lake Insight SQL Syntax Reference.

      -
      • To create a table on the Data Management page:
        1. On the left of the management console, choose Data Management > Databases and Tables.
        2. On the Databases and Tables page, select the database for which you want to create a table. In the Operation column, click More > Create Table to create a table in the current database.
        +
        • To create a table on the Data Management page:
          1. On the left of the management console, choose Data Management > Databases and Tables.
          2. On the Databases and Tables page, select the database for which you want to create a table. In the Operation column, click More > Create Table to create a table in the current database.
        • To create a table on the SQL Editor page:
          1. On the left of the management console, click SQL Editor.
          2. In the navigation pane of the displayed SQL Editor page, click Databases. You can create a table in either of the following ways:
            • Click a database name. In the Tables area, click on the right to create a table in the current database.
            • Click on the right of the database and choose Create Table from the shortcut menu to create a table in the current database.
        @@ -108,7 +108,7 @@

        Type

        Data type of a column. This parameter corresponds to Column Name.

        -
        • string: The data is of the string type.
        • int: Each integer is stored on four bytes.
        • date: The value ranges from 0000-01-01 to 9999-12-31.
        • double: Each number is stored on eight bytes.
        • boolean: Each value is stored on one byte.
        • decimal: The valid bits are positive integers between 1 to 38, including 1 and 38. The decimal digits are integers less than 10.
        • smallint/short: The number is stored on two bytes.
        • bigint/long: The number is stored on eight bytes.
        • timestamp: The data indicates a date and time. The value can be accurate to six decimal points.
        • float: Each number is stored on four bytes.
        • tinyint: Each number is stored on one byte. Only OBS tables support this data type.
        +
        • string: The data is of the string type.
        • int: Each integer is stored on four bytes.
        • date: The value ranges from 0000-01-01 to 9999-12-31.
        • double: Each number is stored on eight bytes.
        • boolean: Each value is stored on one byte.
        • decimal: The valid bits are positive integers between 1 to 38, including 1 and 38. The decimal digits are integers less than 10.
        • smallint/short: The number is stored on two bytes.
        • bigint/long: The number is stored on eight bytes.
        • timestamp: The data indicates a date and time. The value can be accurate to six decimal points.
        • float: Each number is stored on four bytes.
        • tinyint: Each number is stored on one byte. Only OBS tables support this data type.

        string

        @@ -144,7 +144,7 @@

        Data Format

        DLI supports the following data formats:

        -
        • Parquet: DLI can read non-compressed data or data that is compressed using Snappy and gzip.
        • CSV: DLI can read non-compressed data or data that is compressed using gzip.
        • ORC: DLI can read non-compressed data or data that is compressed using Snappy.
        • JSON: DLI can read non-compressed data or data that is compressed using gzip.
        • Avro: DLI can read uncompressed Avro data.
        +
        • Parquet: DLI can read non-compressed data or data that is compressed using Snappy and gzip.
        • CSV: DLI can read non-compressed data or data that is compressed using gzip.
        • ORC: DLI can read non-compressed data or data that is compressed using Snappy.
        • JSON: DLI can read non-compressed data or data that is compressed using gzip.
        • Avro: DLI can read uncompressed Avro data.

        CSV

        @@ -177,7 +177,7 @@

        User-defined Quotation Character

        -

        This parameter is valid only when Data Format is set to CSV and you select User-defined Quotation Character.

        +

        This parameter is valid only when Data Format is set to CSV and you select User-defined Quotation Character.

        The following quotation characters are supported:

        • Single quotation mark (')
        • Double quotation marks (")
        • Others: Enter a user-defined quotation character.
        @@ -213,7 +213,7 @@
    2. -

  2. Click OK.

    After a table is created, you can view and select the table for use on the Data Management page or SQL Editor page.

    +

  3. Click OK.

    After a table is created, you can view and select the table for use on the Data Management page or SQL Editor page.

  4. (Optional) After a DLI table is created, you can decide whether to directly import data to the table.
diff --git a/docs/dli/umn/dli_01_0012.html b/docs/dli/umn/dli_01_0012.html index b756a23c2..314253375 100644 --- a/docs/dli/umn/dli_01_0012.html +++ b/docs/dli/umn/dli_01_0012.html @@ -20,7 +20,7 @@
  • Testing Address Connectivity
    • -
  • Creating a Message Notification Topic
    +
  • Creating an SMN Topic
  • Managing Queue Tags
    • diff --git a/docs/dli/umn/dli_01_0013.html b/docs/dli/umn/dli_01_0013.html index d1a75ff6f..236e45cd5 100644 --- a/docs/dli/umn/dli_01_0013.html +++ b/docs/dli/umn/dli_01_0013.html @@ -20,7 +20,6 @@
  • Method 2: Obtain MRS host information from the /etc/hosts file on an MRS node.
    1. Log in to any MRS node as user root.
    2. Run the following command to obtain MRS hosts information. Copy and save the information.

      cat /etc/hosts

      Figure 1 Obtaining hosts information
    3. Modify host information by referring to Modifying Host Information.
    -
  • Method 3: Log in to FusionInsight Manager to obtain host information.
    1. Log in to FusionInsight Manager.
    2. On FusionInsight Manager, click Hosts. On the Hosts page, obtain the host names and service IP addresses of the MRS hosts.
    3. Modify host information by referring to Modifying Host Information.
    • diff --git a/docs/dli/umn/dli_01_0017.html b/docs/dli/umn/dli_01_0017.html index fb1aa6379..226644b8a 100644 --- a/docs/dli/umn/dli_01_0017.html +++ b/docs/dli/umn/dli_01_0017.html @@ -56,11 +56,10 @@
  • Re-execute: Execute the job again.
  • SparkUI: Display the Spark job execution page.
    NOTE:
    • When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click SparkUI before the cluster is created, an empty projectID will be cached. The SparkUI page cannot be displayed. You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check SparkUI.
    • Currently, only the latest 100 job information records are displayed on the SparkUI of DLI.
    • -
    • In addition to the preceding operations, the following operations are available for QUERY jobs and asynchronous DDL jobs.
      • View Result: View the job running result.
      • Export Result: Export the job running result to the created OBS bucket. For details, see Exporting Query Results.
      +
      • In addition to the preceding operations, the following operations are available for QUERY jobs and asynchronous DDL jobs.
        • View Result: View the job running result.
        • Export Result: Export the job running result to the created OBS bucket.
      • In addition to the preceding operations, the EXPORT job also includes the following operations:
        • Download
        -
      • View Log: Save job logs to the temporary OBS bucket created by DLI.
      • Export Log: Export logs to the created OBS bucket. If the job is in the Running state, logs cannot be exported.
      -
      NOTE:

      To export the logs, you need to obtain the permission to create an OBS bucket.

      -

      Log archiving and export are not available for synchronization jobs and jobs running on the default queue.

      +
    • View Log: Save job logs to the temporary OBS bucket created by DLI.
    +
    NOTE:

    The View Log button is not available for synchronization jobs and jobs running on the default queue.

    @@ -81,7 +80,7 @@

    Exporting Query Results

    A maximum of 1000 records can be displayed in the query result on the console. To view more or all data, you can export the data to OBS. The procedure is as follows:

    You can export results on the SQL Jobs page or the SQL Editor page.

    -
    • On the Job Management > SQL Jobs page, you can click More > Export Result in the Operation column to export the query result.
    • After the query statements are successfully executed on the SQL Editor page, click on the right of the View Result tab page to export the query result.
    +
    • On the Job Management > SQL Jobs page, you can click More > Export Result in the Operation column to export the query result.
    • After the query statements are successfully executed on the SQL Editor page, click next to the View Result tab page to export query results.

    If no column of the numeric type is displayed in the query result, the result cannot be exported.

    diff --git a/docs/dli/umn/dli_01_0318.html b/docs/dli/umn/dli_01_0318.html index dd6cab368..1d7930df5 100644 --- a/docs/dli/umn/dli_01_0318.html +++ b/docs/dli/umn/dli_01_0318.html @@ -109,14 +109,14 @@

    queueActions

    -

    Submitting a job

    +

    Submitting a job (SQL)

    queue

    submitJob

    -

    Canceling a job

    +

    Canceling a job (SQL)

    queue

    diff --git a/docs/dli/umn/dli_01_0320.html b/docs/dli/umn/dli_01_0320.html index 1082bc89f..a48a40c43 100644 --- a/docs/dli/umn/dli_01_0320.html +++ b/docs/dli/umn/dli_01_0320.html @@ -52,66 +52,66 @@

    SQL Editing Window

    SQL job editing window is displayed in the upper right part of the page.

    The SQL statement editing area is below the operation bar. For details about keyboard shortcuts, see Table 3.

    -
    @@ -69,8 +67,6 @@

    Terminating a Job

    On the Spark Jobs page, choose More > Terminate Job in the Operation column of the job that you want to stop.

    -

    Exporting Logs

    On the Spark Jobs page, choose More > Export Log in the Operation column of the corresponding job. In the dialog box that is displayed, enter the path of the created OBS bucket and click OK.

    -
    diff --git a/docs/dli/umn/dli_01_0397.html b/docs/dli/umn/dli_01_0397.html index bf941faf7..a6e22a68f 100644 --- a/docs/dli/umn/dli_01_0397.html +++ b/docs/dli/umn/dli_01_0397.html @@ -2,16 +2,2958 @@

    Built-in Dependencies

    DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar jobs.

    -

    Spark 2.4.5 Dependencies

    • JavaEWAH-1.1.7.jar
    • RoaringBitmap-0.7.45.jar
    • ST4-4.3.1.jar
    • accessors-smart-1.2.jar
    • activation-1.1.1.jar
    • aircompressor-0.16.jar
    • alluxio-2.3.1-luxor-SNAPSHOT-client.jar
    • annotations-17.0.0.jar
    • antlr-2.7.7.jar
    • antlr-runtime-3.4.jar
    • antlr4-runtime-4.8-1.jar
    • aopalliance-1.0.jar
    • aopalliance-repackaged-2.4.0-b34.jar
    • apache-log4j-extras-1.2.17.jar
    • arpack_combined_all-0.1.jar
    • arrow-format-0.12.0.jar
    • arrow-memory-0.12.0.jar
    • arrow-vector-0.12.0.jar
    • asm-5.0.4.jar
    • audience-annotations-0.5.0.jar
    • automaton-1.11-8.jar
    • avro-1.8.2.jar
    • avro-ipc-1.8.2.jar
    • avro-mapred-1.8.2.jar
    • java-sdk-bundle-1.11.856.jar
    • base64-2.3.8.jar
    • bcpkix-jdk15on-1.66.jar
    • bcprov-jdk15on-1.67.jar
    • bonecp-0.8.0.RELEASE.jar
    • breeze-macros_2.11-0.13.2.jar
    • breeze_2.11-0.13.2.jar
    • calcite-avatica-1.2.0-incubating.jar
    • chill-java-0.9.3.jar
    • chill_2.11-0.9.3.jar
    • commons-beanutils-1.9.4.jar
    • commons-cli-1.2.jar
    • commons-codec-1.15.jar
    • commons-collections-3.2.2.jar
    • commons-collections4-4.2.jar
    • commons-compiler-3.0.9.jar
    • commons-compress-1.4.1.jar
    • commons-configuration2-2.1.1.jar
    • commons-crypto-1.0.0-20191105.jar
    • commons-daemon-1.0.13.jar
    • commons-dbcp2-2.7.0.jar
    • commons-httpclient-3.1.jar
    • commons-io-2.5.jar
    • commons-lang-2.6.jar
    • commons-lang3-3.5.jar
    • commons-logging-1.2.jar
    • commons-math3-3.4.1.jar
    • commons-net-3.1.jar
    • commons-pool2-2.8.0.jar
    • commons-text-1.3.jar
    • compress-lzf-1.0.3.jar
    • core-1.1.2.jar
    • crypter-0.0.6.jar
    • curator-client-4.2.0.jar
    • curator-framework-4.2.0.jar
    • curator-recipes-2.7.1.jar
    • datanucleus-api-jdo-3.2.6.jar
    • datanucleus-core-3.2.10.jar
    • datanucleus-rdbms-3.2.9.jar
    • derby-10.14.2.0.jar
    • dnsjava-2.1.7.jar
    • ecj-3.21.0.jar
    • ehcache-3.3.1.jar
    • expiringmap-0.5.9.jar
    • fastutil-8.2.3.jar
    • flatbuffers-java-1.9.0.jar
    • fst-2.50.jar
    • generex-1.0.2.jar
    • geronimo-jcache_1.0_spec-1.0-alpha-1.jar
    • gson-2.2.4.jar
    • guava-14.0.1.jar
    • guice-4.0.jar
    • guice-servlet-4.0.jar
    • hadoop-annotations-3.1.1-ei-302002.jar
    • hadoop-auth-3.1.1-ei-302002.jar
    • hadoop-3.1.1-ei-302002.jar
    • hadoop-client-3.1.1-ei-302002.jar
    • hadoop-common-3.1.1-ei-302002.jar
    • hadoop-hdfs-3.1.1-ei-302002.jar
    • hadoop-hdfs-client-3.1.1-ei-302002.jar
    • hadoop-3.1.1-46.jar
    • hadoop-mapreduce-client-common-3.1.1-ei-302002.jar
    • hadoop-mapreduce-client-core-3.1.1-ei-302002.jar
    • hadoop-mapreduce-client-jobclient-3.1.1-ei-302002.jar
    • hadoop-minikdc-3.1.1-ei-302002.jar
    • hadoop-yarn-api-3.1.1-ei-302002.jar
    • hadoop-yarn-client-3.1.1-ei-302002.jar
    • hadoop-yarn-common-3.1.1-ei-302002.jar
    • hadoop-yarn-registry-3.1.1-ei-302002.jar
    • hadoop-yarn-server-applicationhistoryservice-3.1.1-ei-302002.jar
    • hadoop-yarn-server-common-3.1.1-ei-302002.jar
    • hadoop-yarn-server-resourcemanager-3.1.1-ei-302002.jar
    • hadoop-yarn-server-web-proxy-3.1.1-ei-302002.jar
    • hamcrest-core-1.3.jar
    • hive-common-1.2.1-2.0.0.dli-20220528.090500-402.jar
    • hive-exec-1.2.1-2.0.0.dli-20220528.090521-401.jar
    • hive-metastore-1.2.1-2.0.0.dli-20220528.090509-402.jar
    • hive-shims-0.23-1.2.1-2.0.0.dli-20220528.090445-403.jar
    • hive-shims-1.2.1-2.0.0.dli-20220528.090455-403.jar
    • hive-shims-common-1.2.1-2.0.0.dli-20220528.090441-404.jar
    • hive-shims-scheduler-1.2.1-2.0.0.dli-20220528.090450-403.jar
    • hk2-api-2.4.0-b34.jar
    • hk2-locator-2.4.0-b34.jar
    • hk2-utils-2.4.0-b34.jar
    • hppc-0.7.2.jar
    • htrace-core4-4.2.0-incubating-1.0.0.jar
    • httpclient-4.5.6.jar
    • httpcore-4.4.10.jar
    • ivy-2.4.0.jar
    • jackson-annotations-2.11.4.jar
    • jackson-core-2.11.4.jar
    • jackson-core-asl-1.9.13-atlassian-4.jar
    • jackson-databind-2.11.4.jar
    • jackson-dataformat-yaml-2.11.4.jar
    • jackson-datatype-jsr310-2.11.2.jar
    • jackson-jaxrs-base-2.10.3.jar
    • jackson-jaxrs-json-provider-2.10.3.jar
    • jackson-mapper-asl-1.9.13-atlassian-4.jar
    • jackson-module-jaxb-annotations-2.10.3.jar
    • jackson-module-paranamer-2.11.4.jar
    • jackson-module-scala_2.11-2.11.4.jar
    • jakarta.activation-api-1.2.1.jar
    • jakarta.xml.bind-api-2.3.2.jar
    • janino-3.0.9.jar
    • java-util-1.9.0.jar
    • java-xmlbuilder-1.1.jar
    • javassist-3.18.1-GA.jar
    • javax.annotation-api-1.2.jar
    • javax.inject-1.jar
    • javax.inject-2.4.0-b34.jar
    • javax.servlet-api-3.1.0.jar
    • javax.ws.rs-api-2.0.1.jar
    • javolution-5.3.1.jar
    • jaxb-api-2.2.11.jar
    • jcip-annotations-1.0-1.jar
    • jcl-over-slf4j-1.7.30.jar
    • jdo-api-3.0.1.jar
    • jersey-client-2.23.1.jar
    • jersey-common-2.23.1.jar
    • jersey-container-servlet-2.23.1.jar
    • jersey-container-servlet-core-2.23.1.jar
    • jersey-guava-2.23.1.jar
    • jersey-media-jaxb-2.23.1.jar
    • jersey-server-2.23.1.jar
    • jets3t-0.9.4.jar
    • jettison-1.1.jar
    • jetty-http-9.4.34.v20201102.jar
    • jetty-io-9.4.34.v20201102.jar
    • jetty-security-9.4.34.v20201102.jar
    • jetty-server-9.4.34.v20201102.jar
    • jetty-servlet-9.4.34.v20201102.jar
    • jetty-util-9.4.34.v20201102.jar
    • jetty-util-ajax-9.4.34.v20201102.jar
    • jetty-webapp-9.4.34.v20201102.jar
    • jetty-xml-9.4.34.v20201102.jar
    • joda-time-2.9.3.jar
    • jodd-core-3.5.2.jar
    • json-20200518.jar
    • json-io-2.5.1.jar
    • json-sanitizer-1.2.1.jar
    • json-smart-2.3.jar
    • json4s-ast_2.11-3.5.3.jar
    • json4s-core_2.11-3.5.3.jar
    • json4s-jackson_2.11-3.5.3.jar
    • json4s-scalap_2.11-3.5.3.jar
    • jsp-api-2.1.jar
    • jsr305-1.3.9.jar
    • jta-1.1.jar
    • jtransforms-2.4.0.jar
    • jts-core-1.16.1.jar
    • jul-to-slf4j-1.7.30.jar
    • junit-4.11.jar
    • kerb-admin-1.0.1.jar
    • kerb-client-1.0.1.jar
    • kerb-common-1.0.1.jar
    • kerb-core-1.0.1.jar
    • kerb-crypto-1.0.1.jar
    • kerb-identity-1.0.1.jar
    • kerb-server-1.0.1.jar
    • kerb-simplekdc-1.0.1.jar
    • kerb-util-1.0.1.jar
    • kerby-asn1-1.0.1.jar
    • kerby-config-1.0.1.jar
    • kerby-pkix-1.0.1.jar
    • kerby-util-1.0.1.jar
    • kerby-xdr-1.0.1.jar
    • kryo-shaded-4.0.2.jar
    • kubernetes-client-5.4.1-20211025.jar
    • kubernetes-model-admissionregistration-5.4.1-20211025.jar
    • kubernetes-model-apiextensions-5.4.1-20211025.jar
    • kubernetes-model-apps-5.4.1-20211025.jar
    • kubernetes-model-autoscaling-5.4.1-20211025.jar
    • kubernetes-model-batch-5.4.1-20211025.jar
    • kubernetes-model-certificates-5.4.1-20211025.jar
    • kubernetes-model-common-5.4.1-20211025.jar
    • kubernetes-model-coordination-5.4.1-20211025.jar
    • kubernetes-model-core-5.4.1-20211025.jar
    • kubernetes-model-discovery-5.4.1-20211025.jar
    • kubernetes-model-events-5.4.1-20211025.jar
    • kubernetes-model-extensions-5.4.1-20211025.jar
    • kubernetes-model-flowcontrol-5.4.1-20211025.jar
    • kubernetes-model-metrics-5.4.1-20211025.jar
    • kubernetes-model-networking-5.4.1-20211025.jar
    • kubernetes-model-node-5.4.1-20211025.jar
    • kubernetes-model-policy-5.4.1-20211025.jar
    • kubernetes-model-rbac-5.4.1-20211025.jar
    • kubernetes-model-scheduling-5.4.1-20211025.jar
    • kubernetes-model-storageclass-5.4.1-20211025.jar
    • leveldbjni-all-1.8-20191105.jar
    • libfb303-0.9.3.jar
    • libthrift-0.12.0.jar
    • log4j-1.2.17-cloudera1.jar
    • log4j-rolling-appender-20131024-2017.jar
    • logging-interceptor-3.14.9.jar
    • lucene-analyzers-common-7.7.2.jar
    • lucene-core-7.7.2.jar
    • lucene-queryparser-7.7.2.jar
    • lucene-sandbox-7.7.2.jar
    • luxor-encrypt-2.0.0-20220623.010726-213.jar
    • luxor-fs3-2.0.0-20220623.010750-209.jar
    • luxor-obs-fs3-2.0.0-20220623.010756-209.jar
    • luxor-rpc_2.11-2.0.0-20220623.010737-182.jar
    • luxor-transport-2.0.0-20220623.010744-71.jar
    • lz4-java-1.7.1.jar
    • machinist_2.11-0.6.1.jar
    • macro-compat_2.11-1.1.1.jar
    • metrics-core-3.1.5.jar
    • metrics-graphite-3.1.5.jar
    • metrics-jmx-4.1.12.1.jar
    • metrics-json-3.1.5.jar
    • metrics-jvm-3.1.5.jar
    • minlog-1.3.0.jar
    • mssql-jdbc-6.2.1.jre7.jar
    • netty-all-4.1.51.Final.jar
    • nimbus-jose-jwt-8.19.jar
    • objenesis-2.5.1.jar
    • okhttp-3.14.9.jar
    • okio-1.17.2.jar
    • opencsv-2.3.jar
    • opencsv-4.6.jar
    • opencv-4.3.0-2.jar
    • orc-core-1.6.8-nohive.jar
    • orc-mapreduce-1.6.8-nohive.jar
    • orc-shims-1.6.8.jar
    • oro-2.0.8.jar
    • osgi-resource-locator-1.0.1.jar
    • paranamer-2.8.jar
    • parquet-column-1.12.2.jar
    • parquet-common-1.12.2.jar
    • parquet-encoding-1.12.2.jar
    • parquet-format-structures-1.12.2.jar
    • parquet-hadoop-1.12.2.jar
    • parquet-hadoop-bundle-1.6.0.jar
    • parquet-jackson-1.12.2.jar
    • postgresql-42.2.14.jar
    • protobuf-java-2.5.0.jar
    • py4j-0.10.7.jar
    • pyrolite-4.13.jar
    • re2j-1.1.jar
    • scala-compiler-2.11.12.jar
    • scala-library-2.11.12.jar
    • scala-parser-combinators_2.11-1.1.2.jar
    • scala-reflect-2.11.12.jar
    • scala-xml_2.11-1.0.5.jar
    • secComponentApi-1.0.6.jar
    • shapeless_2.11-2.3.2.jar
    • shims-0.7.45.jar
    • slf4j-api-1.7.30.jar
    • slf4j-log4j12-1.7.30.jar
    • snakeyaml-1.26.jar
    • snappy-java-1.1.8.2.jar
    • solr-core-7.7.2.jar
    • solr-solrj-7.7.2.jar
    • spark-avro_2.11-2.4.5.0100-2.0.0.dli-20220617.085536-9.jar
    • spark-avro_2.11-4.0.0.jar
    • spark-catalyst_2.11-2.4.5.0100-2.0.0.dli-20220617.085405-16.jar
    • spark-core_2.11-2.4.5.0100-2.0.0.dli-20220617.085327-16.jar
    • spark-graphx_2.11-2.4.5.0100-.0.dli-20220617.085336-16.jar
    • spark-hive_2.11-2.4.5.0100-2.0.0.dli-20220617.085423-16.jar
    • spark-kubernetes_2.11-2.4.5.0100-2.0.0.dli-20220617.085519-16.jar
    • spark-kvstore_2.11-2.4.5.0100-2.0.0.dli-20220617.085249-16.jar
    • spark-launcher_2.11-2.4.5.0100-2.0.0.dli-20220617.085435-16.jar
    • spark-mllib-local_2.11-2.4.5.0100-2.0.0.dli-20220617.085349-16.jar
    • spark-mllib_2.11-2.4.5.0100-2.0.0.dli-20220617.085342-16.jar
    • spark-network-common_2.11-2.4.5.0100-2.0.0.dli-20220617.085254-16.jar
    • spark-network-shuffle_2.11-2.4.5.0100-2.0.0.dli-20220617.085300-16.jar
    • spark-om_2.11-2.4.5.0100-2.0.0.dli-20220617.085316-16.jar
    • spark-repl_2.11-2.4.5.0100-2.0.0.dli-20220617.085430-16.jar
    • spark-sketch_2.11-2.4.5.0100-2.0.0.dli-20220617.085243-16.jar
    • spark-sql_2.11-2.4.5.0100-2.0.0.dli-20220617.085414-16.jar
    • spark-streaming_2.11-2.4.5.0100-.0.dli-20220617.085359-16.jar
    • spark-tags_2.11-2.4.5.0100-2.0.0.dli-20220617.085322-16.jar
    • spark-unsafe_2.11-2.4.5.0100-2.0.0.dli-20220617.085311-16.jar
    • spark-uquery_2.11-2.4.5.0100-2.0.0.dli-SNAPSHOT.jar
    • spark-yarn_2.11-2.4.5.0100-2.0.0.dli-20220617.085531-16.jar
    • spire-macros_2.11-0.13.0.jar
    • spire_2.11-0.13.0.jar
    • stax-api-1.0-2.jar
    • stax2-api-3.1.4.jar
    • stream-2.7.0.jar
    • stringtemplate-3.2.1.jar
    • threeten-extra-1.5.0.jar
    • tink-1.6.0.jar
    • token-provider-1.0.1.jar
    • tomcat-api-9.0.39.jar
    • tomcat-el-api-9.0.39.jar
    • tomcat-jasper-9.0.39.jar
    • tomcat-jasper-el-9.0.39.jar
    • tomcat-jsp-api-9.0.39.jar
    • tomcat-juli-9.0.39.jar
    • tomcat-servlet-api-9.0.39.jar
    • tomcat-util-9.0.39.jar
    • tomcat-util-scan-9.0.39.jar
    • univocity-parsers-2.7.3.jar
    • validation-api-1.1.0.Final.jar
    • velocity-1.7.jar
    • woodstox-core-5.0.3.jar
    • xbean-asm6-shaded-4.8.jar
    • xercesImpl-2.12.0.jar
    • xml-apis-1.4.01.jar
    • xz-1.0.jar
    • zjsonpatch-0.3.0.jar
    • zookeeper-3.5.6-ei-302002.jar
    • zookeeper-jute-3.5.6-ei-302002.jar
    • zstd-jni-1.4.9-1.jar
    +

    Spark 3.1.1 Dependencies

    +
    Table 2 Components of the SQL job editing window

    No.

    +
    - - - - - - - - - - - - - - - - - - - - - - - @@ -176,9 +176,7 @@

    View Result: View the execution result of a QUERY job.

    Export Result: Export the execution results of a QUERY job to a specified OBS path.

    View Log: View the OBS path for storing SQL statement execution logs.

    -

    Export Log: Export SQL statement execution logs.

    -
    NOTE:

    To export the logs, you need to obtain the permission to create an OBS bucket.

    -

    View Log and Export Log buttons are not available for synchronization jobs and jobs running on the default queue.

    +
    NOTE:

    The View Log button is not available for synchronization jobs and jobs running on the default queue.

    @@ -209,8 +207,7 @@
    - diff --git a/docs/dli/umn/dli_01_0363.html b/docs/dli/umn/dli_01_0363.html index aae9fabda..1fb0600d8 100644 --- a/docs/dli/umn/dli_01_0363.html +++ b/docs/dli/umn/dli_01_0363.html @@ -2,55 +2,54 @@

    Creating a Queue

    Before executing a job, you need to create a queue.

    -
    • If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main account and keep records in the DLI database before creating a queue.
    • It takes 6 to 10 minutes for a job running on a new queue for the first time.
    • After a queue is created, if no job is run within one hour, the system releases the queue.
    +
    • If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main account and keep records in the DLI database before creating a queue.
    • It takes 6 to 10 minutes for a job running on a new queue for the first time.
    • After a queue is created, if no job is run within one hour, the system releases the queue.
    • Queues with 16 CUs do not support scale-out or scale-in.
    • Queues with 64 CUs do not support scale-in.

    Procedure

    1. You can create a queue on the Overview, SQL Editor, or Queue Management page.
      • In the upper right corner of the Overview page, click Create Queue.
      • To create a queue on the Queue Management page:
        1. In the navigation pane of the DLI management console, choose Resources >Queue Management.
        2. In the upper right corner of the Queue Management page, click Create Queue to create a queue.
      • To create a queue on the SQL Editor page:
        1. In the navigation pane of the DLI management console, click SQL Editor.
        2. Click Queues. On the tab page displayed, click on the right to create a queue.
      -
    2. In the displayed Create Queue dialog box, set related parameters by referring to Table 1.
      -
    Table 2 Components of the SQL job editing window

    No.

    Button & Drop-Down List

    +

    Button & Drop-Down List

    Description

    +

    Description

    2

    +

    2

    Queues

    +

    Queues

    Select a queue from the drop-down list box. If no queue is available, the default queue is displayed. Refer to Creating a Queue and create a queue.

    +

    Select a queue from the drop-down list box. If no queue is available, the default queue is displayed. Refer to Creating a Queue and create a queue.

    SQL jobs can be executed only on SQL queues.

    3

    +

    3

    Database

    +

    Database

    Select a database from the drop-down list box. If no database is available, the default database is displayed. For details about how to create a database, see Creating a Database or a Table.

    +

    Select a database from the drop-down list box. If no database is available, the default database is displayed. For details about how to create a database, see Creating a Database or a Table.

    NOTE:

    If you specify the database in the SQL statements, the database you choose from the drop-down list will not be used.

    4

    +

    4

    Execute

    +

    Execute

    Click this button to run the SQL statements in the job editing window.

    +

    Click this button to run the SQL statements in the job editing window.

    5

    +

    5

    Format

    +

    Format

    Click this button to format the SQL statements.

    +

    Click this button to format the SQL statements.

    6

    +

    6

    Syntax Reference

    +

    Syntax Reference

    Click this button to view the Data Lake Insight SQL Syntax Reference.

    +

    Click this button to view the Data Lake Insight SQL Syntax Reference.

    7

    +

    7

    Settings

    +

    Settings

    Add parameters and tags.

    +

    Add parameters and tags.

    Parameter Settings: Set parameters in key/value format for SQL jobs.

    Tags: Set tags in key/value format for SQL jobs.

    8

    +

    8

    More

    +

    More

    The drop-down list includes the following options:

    +

    The drop-down list includes the following options:

    • Click Verify Syntax to check whether the SQL statements are correct.
    • Click Set as Template to set SQL statements as a template. For details, see Managing SQL Templates.
    • Click Change Theme to switch between dark and light modes.

    Export the result

    Click to export the query result to OBS. For details, see Exporting Query Results.

    -

    A maximum of 1000 records can be displayed in the query result on the console. To view more or all data, you can click Export Result to export the data to OBS.

    +

    Export the job execution results to the created OBS bucket. For details, see SQL Job Management.
    @@ -137,7 +138,14 @@
    Table 1 Parameters

    Parameter

    +
  • On the Create Queue page displayed, set the parameters according to Table 1. +
    - - - - - - - - - - - - -
    Table 1 Parameters

    Parameter

    Description

    +

    Description

    Name

    +

    Name

    Name of a queue.

    +

    Name of a queue.

    • The queue name can contain only digits, letters, and underscores (_), but cannot contain only digits, start with an underscore (_), or be left unspecified.
    • The length of the name cannot exceed 128 characters.
    NOTE:

    The queue name is case-insensitive. Uppercase letters will be automatically converted to lowercase letters.

    Type

    +

    Type

    • For SQL: compute resources used for SQL jobs.
    • For general purpose: compute resources used for Spark and Flink jobs.
      NOTE:

      Selecting Dedicated Resource Mode enables you to create a dedicated queue. Enhanced datasource connections can only be created for dedicated queues.

      +
    • For SQL: compute resources used for SQL jobs.
    • For general purpose: compute resources used for Spark and Flink jobs.
      NOTE:

      Selecting Dedicated Resource Mode enables you to create a dedicated queue. Enhanced datasource connections can only be created for dedicated queues.

    Specifications

    +

    Specifications

    Select queue specifications as required. A CU includes one core and 4 GB memory. You can set the total number of CUs on all compute nodes of a queue. DLI automatically allocates the memory and vCPUs for each node.

    -
    • Fixed specifications include 16 CUs, 64 CUs, 256 CUs, and 512 CUs.
    • Custom: Set the specifications as required.
    +

    The compute nodes' total number of CUs. One CU equals one vCPU and 4 GB of memory. DLI automatically assigns CPU and memory resources to each compute node, and the client does not need to know how many compute nodes are being used.

    Description

    +

    Description

    Description of the queue to be created. The description can contain a maximum of 128 characters.

    +

    Description of the queue to be created. The description can contain a maximum of 128 characters.

    Advanced Settings

    +

    Advanced Settings

    In the Queue Type area, select Dedicated Resource Mode and then click Advanced Settings.
    • Default: The system automatically configures the parameter.
    • Custom

      CIDR Block: You can specify the CIDR block. For details, see Modifying the Queue CIDR Block. If DLI enhanced datasource connection is used, the CIDR block of the DLI queue cannot overlap with that of the data source.

      +
    In the Queue Type area, select Dedicated Resource Mode and then click Advanced Settings.
    • Default: The system automatically configures the parameter.
    • Custom

      CIDR Block: You can specify the CIDR block. For details, see Modifying the CIDR Block. If DLI enhanced datasource connection is used, the CIDR block of the DLI queue cannot overlap with that of the data source.

      Queue Type: When running an AI-related SQL job, select AI-enhanced. When running other jobs, select Basic.

    Tag

    +

    Tags

    Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

    +

    Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

    NOTE:
    • A maximum of 20 tags can be added.
    • Only one tag value can be added to a tag key.
    • The key name in each resource must be unique.
    • Tag key: Enter a tag key name in the text box.
      NOTE:

      A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

      @@ -63,7 +62,6 @@
    -
  • Click Create Now to create a queue.

    After a queue is created, you can view and select the queue for use on the Queue Management page.

    It takes 6 to 10 minutes for a job running on a new queue for the first time.

    diff --git a/docs/dli/umn/dli_01_0378.html b/docs/dli/umn/dli_01_0378.html index 10d7c3c5f..5af837a1f 100644 --- a/docs/dli/umn/dli_01_0378.html +++ b/docs/dli/umn/dli_01_0378.html @@ -5,9 +5,9 @@

    DLI supports standard SQL and is compatible with Spark SQL and Flink SQL. It also supports multiple access modes, and is compatible with mainstream data formats. DLI supports SQL statements and Spark applications for heterogeneous data sources, including CloudTable, RDS, GaussDB(DWS), CSS, OBS, custom databases on ECSs, and offline databases.

    Functions

    You can query and analyze heterogeneous data sources such as RDS, and GaussDB(DWS) on the cloud using access methods, such as visualized interface, RESTful API, JDBC, and Beeline. The data format is compatible with five mainstream data formats: CSV, JSON, Parquet, and ORC.

    -
    • Basic functions
      • You can use standard SQL statements to query in SQL jobs. For details, see .
      • Flink jobs support Flink SQL online analysis. Aggregation functions such as Window and Join, geographic functions, and CEP functions are supported. SQL is used to express service logic, facilitating service implementation. For details, see .
      • For spark jobs, fully-managed Spark computing can be performed. You can submit computing tasks through interactive sessions or in batch to analyze data in the fully managed Spark queues. For details, see .
      -
    • Federated analysis of heterogeneous data sources
      • Spark datasource connection: Data sources such as DWS, RDS, and CSS can be accessed through DLI. For details, see .
      • Interconnection with multiple cloud services is supported in Flink jobs to form a rich stream ecosystem. The DLI stream ecosystem consists of cloud service ecosystems and open source ecosystems.
        • Cloud service ecosystem: DLI can interconnect with other services in Flink SQL. You can directly use SQL to read and write data from cloud services.
        • Open-source ecosystems: After connections to other VPCs are established through datasource connections, you can access all data sources and output targets (such as Kafka, HBase, and Elasticsearch) supported by Flink and Spark in your dedicated DLI queue.
        -

        For details, see .

        +
        • Basic functions
          • You can use standard SQL statements to query in SQL jobs.
          • Flink jobs support Flink SQL online analysis. Aggregation functions such as Window and Join, geographic functions, and CEP functions are supported. SQL is used to express service logic, facilitating service implementation.
          • For spark jobs, fully-managed Spark computing can be performed. You can submit computing tasks through interactive sessions or in batch to analyze data in the fully managed Spark queues.
          +
        • Federated analysis of heterogeneous data sources
          • Spark datasource connection: Data sources such as DWS, RDS, and CSS can be accessed through DLI.
          • Interconnection with multiple cloud services is supported in Flink jobs to form a rich stream ecosystem. The DLI stream ecosystem consists of cloud service ecosystems and open source ecosystems.
            • Cloud service ecosystem: DLI can interconnect with other services in Flink SQL. You can directly use SQL to read and write data from cloud services.
            • Open-source ecosystems: After connections to other VPCs are established through datasource connections, you can access all data sources and output targets (such as Kafka, HBase, and Elasticsearch) supported by Flink and Spark in your dedicated DLI queue.
            +

        • Storage-compute decoupling

          DLI is interconnected with OBS for data analysis. In this architecture where storage and compute are decoupled, resources of these two types are charged separately, helping you reduce costs and improving resource utilization.

          You can choose single-AZ or multi-AZ storage when you create an OBS bucket for storing redundant data on the DLI console. The differences between the two storage policies are as follows:

          @@ -20,10 +20,11 @@
          • Auto scaling: DLI ensures you always have enough capacity on hand to deal with any traffic spikes.

    Accessing DLI

    A web-based service management platform is provided. You can access DLI using the management console or HTTPS-based APIs, or connect to the DLI server through the JDBC client.

    -
    • Using the management console

      You can submit SQL, Spark, or Flink jobs on the DLI management console. Log in to the management console. Choose EI Enterprise Intelligence > Data Lake Insight.

      +
      • Using the management console

        You can submit SQL, Spark, or Flink jobs on the DLI management console.

        +

        Log in to the management console and choose Data Analysis > Data Lake Insight.

      • Using APIs

        If you need to integrate DLI into a third-party system for secondary development, you can call DLI APIs to use the service.

        -

        For details, see Data Lake Insight API Reference.

        +

        For details, see Data Lake Insight API Reference.

    diff --git a/docs/dli/umn/dli_01_0384.html b/docs/dli/umn/dli_01_0384.html index 4dc6e7d24..aaa00c550 100644 --- a/docs/dli/umn/dli_01_0384.html +++ b/docs/dli/umn/dli_01_0384.html @@ -5,7 +5,7 @@

    On the Overview page, click Create Job in the upper right corner of the Spark Jobs tab or click Create Job in the upper right corner of the Spark Jobs page. The Spark job editing page is displayed.

    On the Spark job editing page, a message is displayed, indicating that a temporary DLI data bucket will be created. The created bucket is used to store temporary data generated by DLI, such as job logs and job results. You cannot view job logs if you choose not to create it. The bucket will be created and the default bucket name is used.

    If you do not need to create a DLI temporary data bucket and do not want to receive this message, select Do not show again and click Cancel.

    -

    Prerequisites

    • You have uploaded the dependencies to the corresponding OBS bucket on the Data Management > Package Management page. For details, see Creating a Package.
    • Before creating a Spark job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS MongoDB, you need to create a cross-source connection to enable the network between the job running queue and external data sources.
      • For details about the external data sources that can be accessed by Spark jobs, see Cross-Source Analysis Development Methods.
      • For details about how to create a datasource connection, see Enhanced Datasource Connections.

        On the Resources > Queue Management page, locate the queue you have created, and choose More > Test Address Connectivity in the Operation column to check whether the network connection between the queue and the data source is normal. For details, see Testing Address Connectivity.

        +

        Prerequisites

        • You have uploaded the dependencies to the corresponding OBS bucket on the Data Management > Package Management page. For details, see Creating a Package.
        • Before creating a Spark job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a cross-source connection to enable the network between the job running queue and external data sources.
          • For details about the external data sources that can be accessed by Spark jobs, see Cross-Source Analysis Development Methods.
          • For details about how to create a datasource connection, see Enhanced Datasource Connections.

            On the Resources > Queue Management page, locate the queue you have created, and choose More > Test Address Connectivity in the Operation column to check whether the network connection between the queue and the data source is normal. For details, see Testing Address Connectivity.

        @@ -28,6 +28,7 @@

    • Application

    Select the package to be executed. The value can be .jar or .py.

    +

    You can select the name of a JAR or pyFile package that has been uploaded to the DLI resource management system. You can also specify an OBS path, for example, obs://Bucket name/Package name.

    Main Class (--class)

    @@ -38,8 +39,8 @@

    Spark Arguments (--conf)

    Enter a parameter in the format of key=value. Press Enter to separate multiple key-value pairs.

    -

    These parameters can be replaced using global variables. For example, if you create a global variable custom_class on the Global Configuration > Global Variables page, you can use "spark.sql.catalog"={{custom_class}} to replace a parameter with this variable after the job is submitted.

    -
    NOTE:
    • The JVM garbage collection algorithm cannot be customized for Spark jobs.
    • If the Spark version is 3.1.1, configure Spark parameters (--conf) to select a dependent module. For details about the configuration, see Creating a Spark Job.
    +

    These parameters can be replaced with global variables. For example, if you create a global variable custom_class on the Global Configuration > Global Variables page, you can use "spark.sql.catalog"={{custom_class}} to replace a parameter with this variable after the job is submitted.

    +
    NOTE:
    • The JVM garbage collection algorithm cannot be customized for Spark jobs.
    • If the Spark version is 3.1.1, configure Spark parameters (--conf) to select a dependent module. For details about the example configuration, see Table 2.
    -
  • Set the following parameters in advanced settings:
    • Select Dependency Resources: For details about the parameters, see Table 3.
    • Configure Resources: For details about the parameters, see Table 4.
    +
  • Set the following parameters in advanced settings:
    • Select Dependency Resources: For details about the parameters, see Table 3.
    • Configure Resources: For details about the parameters, see Table 4.

      The parallelism degree of Spark resources is jointly determined by the number of Executors and the number of Executor CPU cores.

      +

      Maximum number of tasks that can be concurrently executed = Number of Executors x Number of Executor CPU cores

      +

      You can properly plan compute resource specifications based on the compute CUs of the queue you have purchased.

      +

      Note that Spark tasks need to be jointly executed by multiple roles, such as driver and executor. So, the number of executors multiplied by the number of executor CPU cores must be less than the number of compute CUs of the queue to prevent other roles from failing to start Spark tasks. For more information about roles for Spark tasks, see Apache Spark.

      +

      Calculation formula for Spark job parameters:

      +
      • CUs = Driver Cores + Executors x Executor Cores
      • Memory = Driver Memory + (Executors x Executor Memory)
      +
      +
    @@ -169,7 +177,8 @@ - @@ -177,35 +186,32 @@ - - - -
    Table 3 Parameters for selecting dependency resources

    Parameter

    Resource Specifications

    Select a resource specification from the drop-down list box. The system provides three resource specifications for you to select. The following configuration items in the resource specifications can be modified:

    +

    Select a resource specification from the drop-down list box. The system provides three resource specification options for you to choose from.

    +

    Resource specifications involve the following parameters:

    • Executor Memory
    • Executor Cores
    • Executors
    • Driver Cores
    • Driver Memory

    If modified, your modified settings of the items are used.

    Executor Memory

    Customize the configuration item based on the selected resource specifications.

    +

    Memory of each Executor. It is recommended that the ratio of Executor CPU cores to Executor memory be 1:4.

    Executor Cores

    Customize the configuration item based on the selected resource specifications.

    +

    Number of CPU cores of each Executor applied for by Spark jobs, which determines the capability of each Executor to execute tasks concurrently.

    Executors

    Customize the configuration item based on the selected resource specifications.

    +

    Number of Executors applied for by a Spark job

    Driver Cores

    Customize the configuration item based on the selected resource specifications.

    +

    Number of CPU cores of the driver

    Driver Memory

    Customize the configuration item based on the selected resource specifications.

    +

    Driver memory size. It is recommended that the ratio of the number of driver CPU cores to the driver memory be 1:4.
    -

    Spark job parameter calculation:

    -
    • Number of CUs = Number of driver CPU cores + Number of executors x Number of executor CPU cores

      The cluster management plane and driver use some CU resources. Number of Executors * Number of Executor Cores must be smaller than the number of computing CUs of the queue.

      -
    • Memory = Driver memory + (Number of Executors x Executor memory)
    -
  • Click Execute in the upper right corner of the Spark job editing page.

    After the message "Batch processing job submitted successfully" is displayed, you can view the status and logs of the submitted job on the Spark Jobs page.

    • diff --git a/docs/dli/umn/dli_01_0385.html b/docs/dli/umn/dli_01_0385.html index 265cabc18..f99ca6c6a 100644 --- a/docs/dli/umn/dli_01_0385.html +++ b/docs/dli/umn/dli_01_0385.html @@ -54,9 +54,7 @@
    • Edit: You can modify the current job configuration and re-execute the job.
    • SparkUI: After you click this button, the Spark job execution page is displayed.
      NOTE:
      • The SparkUI page cannot be viewed for jobs in the Starting state.
      • Currently, only the latest 100 job information records are displayed on the SparkUI of DLI.
      -
    • Terminate Job: Cancel a job that is being started or running.
    • Re-execute: Run the job again.
    • Archive Log: Save job logs to the temporary bucket created by DLI.
    • Export Log: Export logs to the created OBS bucket.
      NOTE:
      • You have the permission to create OBS buckets.
      • If the job is in the Running state, logs cannot be exported.
      -
      -
    • Commit Log: View the logs of submitted jobs.
    • Driver Log: View the logs of running jobs.
    +
  • Terminate Job: Cancel a job that is being started or running.
  • Re-execute: Run the job again.
  • Archive Log: Save job logs to the temporary bucket created by DLI.
  • Commit Log: View the logs of submitted jobs.
  • Driver Log: View the logs of running jobs.
    		•
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 1 Spark 3.1.1 dependencies

    Dependency

    +

    accessors-smart-1.2.jar

    +

    hive-shims-scheduler-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    metrics-graphite-4.1.1.jar

    +

    activation-1.1.1.jar

    +

    hive-spark-client-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    metrics-jmx-4.1.1.jar

    +

    aggdesigner-algorithm-6.0.jar

    +

    hive-standalone-metastore-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    metrics-json-4.1.1.jar

    +

    aircompressor-0.16.jar

    +

    hive-storage-api-2.7.2.jar

    +

    metrics-jvm-4.1.1.jar

    +

    algebra_2.12-2.0.0-M2.jar

    +

    hive-vector-code-gen-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    minlog-1.3.0.jar

    +

    annotations-17.0.0.jar

    +

    hk2-api-2.6.1.jar

    +

    netty-3.10.6.Final.jar

    +

    ant-1.10.9.jar

    +

    hk2-locator-2.6.1.jar

    +

    netty-all-4.1.86.Final.jar

    +

    ant-launcher-1.10.9.jar

    +

    hk2-utils-2.6.1.jar

    +

    netty-buffer-4.1.86.Final.jar

    +

    antlr4-runtime-4.8-1.jar

    +

    hppc-0.7.2.jar

    +

    netty-codec-4.1.86.Final.jar

    +

    antlr-runtime-3.5.2.jar

    +

    httpclient-4.5.6.jar

    +

    netty-codec-dns-4.1.86.Final.jar

    +

    aopalliance-1.0.jar

    +

    httpcore-4.4.10.jar

    +

    netty-codec-haproxy-4.1.86.Final.jar

    +

    aopalliance-repackaged-2.6.1.jar

    +

    istack-commons-runtime-3.0.8.jar

    +

    netty-codec-http2-4.1.86.Final.jar

    +

    apiguardian-api-1.1.0.jar

    +

    ivy-2.5.0.jar

    +

    netty-codec-http-4.1.86.Final.jar

    +

    arpack_combined_all-0.1.jar

    +

    jackson-annotations-2.13.2.jar

    +

    netty-codec-memcache-4.1.86.Final.jar

    +

    arrow-format-2.0.0.jar

    +

    jackson-core-2.13.2.jar

    +

    netty-codec-mqtt-4.1.86.Final.jar

    +

    arrow-memory-core-2.0.0.jar

    +

    jackson-core-asl-1.9.13-atlassian-4.jar

    +

    netty-codec-redis-4.1.86.Final.jar

    +

    arrow-memory-netty-2.0.0.jar

    +

    jackson-databind-2.13.2.2.jar

    +

    netty-codec-smtp-4.1.86.Final.jar

    +

    arrow-vector-2.0.0.jar

    +

    jackson-dataformat-yaml-2.13.2.jar

    +

    netty-codec-socks-4.1.86.Final.jar

    +

    asm-5.0.4.jar

    +

    jackson-datatype-jsr310-2.11.2.jar

    +

    netty-codec-stomp-4.1.86.Final.jar

    +

    audience-annotations-0.5.0.jar

    +

    jackson-mapper-asl-1.9.13-atlassian-4.jar

    +

    netty-codec-xml-4.1.86.Final.jar

    +

    automaton-1.11-8.jar

    +

    jackson-module-jaxb-annotations-2.13.2.jar

    +

    netty-common-4.1.86.Final.jar

    +

    avatica-1.22.0.jar

    +

    jackson-module-scala_2.12-2.13.2.jar

    +

    netty-handler-4.1.86.Final.jar

    +

    avatica-core-1.16.0.jar

    +

    jaeger-client-1.6.0.jar

    +

    netty-handler-proxy-4.1.86.Final.jar

    +

    avatica-metrics-1.16.0.jar

    +

    jaeger-core-1.6.0.jar

    +

    netty-handler-ssl-ocsp-4.1.86.Final.jar

    +

    avatica-server-1.16.0.jar

    +

    jaeger-thrift-1.6.0.jar

    +

    netty-resolver-4.1.86.Final.jar

    +

    avro-1.8.2.jar

    +

    jaeger-tracerresolver-1.6.0.jar

    +

    netty-resolver-dns-4.1.86.Final.jar

    +

    avro-ipc-1.8.2.jar

    +

    jakarta.activation-api-1.2.1.jar

    +

    netty-resolver-dns-classes-macos-4.1.86.Final.jar

    +

    avro-mapred-1.8.2.jar

    +

    jakarta.annotation-api-1.3.5.jar

    +

    netty-resolver-dns-native-macos-4.1.86.Final-osx-aarch_64.jar

    +

    java-sdk-bundle-1.11.856.jar

    +

    jakarta.el-3.0.3.jar

    +

    netty-resolver-dns-native-macos-4.1.86.Final-osx-x86_64.jar

    +

    base64-2.3.8.jar

    +

    jakarta.el-api-3.0.3.jar

    +

    netty-transport-4.1.86.Final.jar

    +

    bcpkix-jdk15on-1.69.jar

    +

    jakarta.inject-2.6.1.jar

    +

    netty-transport-classes-epoll-4.1.86.Final.jar

    +

    bcprov-jdk15on-1.69.jar

    +

    jakarta.servlet-api-4.0.3.jar

    +

    netty-transport-classes-kqueue-4.1.86.Final.jar

    +

    bcutil-jdk15on-1.69.jar

    +

    jakarta.validation-api-2.0.2.jar

    +

    netty-transport-native-epoll-4.1.86.Final-linux-aarch_64.jar

    +

    bonecp-0.8.0.RELEASE.jar

    +

    jakarta.ws.rs-api-2.1.6.jar

    +

    netty-transport-native-epoll-4.1.86.Final-linux-x86_64.jar

    +

    breeze_2.12-1.0.jar

    +

    jakarta.xml.bind-api-2.3.2.jar

    +

    netty-transport-native-kqueue-4.1.86.Final-osx-aarch_64.jar

    +

    breeze-macros_2.12-1.0.jar

    +

    jamon-runtime-2.4.1.jar

    +

    netty-transport-native-kqueue-4.1.86.Final-osx-x86_64.jar

    +

    caffeine-2.8.1.jar

    +

    janino-3.0.16.jar

    +

    netty-transport-native-unix-common-4.1.86.Final.jar

    +

    calcite-core-1.22.0.jar

    +

    JavaEWAH-0.3.2.jar

    +

    netty-transport-rxtx-4.1.86.Final.jar

    +

    calcite-druid-1.19.0.jar

    +

    java-sdk-core-3.0.12.jar

    +

    netty-transport-sctp-4.1.86.Final.jar

    +

    calcite-linq4j-1.22.0.jar

    +

    javassist-3.25.0-GA.jar

    +

    netty-transport-udt-4.1.86.Final.jar

    +

    cats-kernel_2.12-2.0.0-M4.jar

    +

    javax.activation-api-1.2.0.jar

    +

    nimbus-jose-jwt-8.19.jar

    +

    checker-qual-3.5.0.jar

    +

    javax.annotation-api-1.3.2.jar

    +

    objenesis-2.5.1.jar

    +

    chill_2.12-0.9.5.jar

    +

    javax.inject-1.jar

    +

    okhttp-3.14.9.jar

    +

    chill-java-0.9.5.jar

    +

    javax.jdo-3.2.0-m3.jar

    +

    okio-1.17.2.jar

    +

    classmate-1.5.1.jar

    +

    java-xmlbuilder-1.1.jar

    +

    opencsv-2.3.jar

    +

    commons-beanutils-1.9.4.jar

    +

    javax.servlet-api-3.1.0.jar

    +

    opentelemetry-api-1.16.0.jar

    +

    commons-cli-1.2.jar

    +

    javax.transaction-api-1.3.jar

    +

    opentelemetry-context-1.16.0.jar

    +

    commons-codec-1.15.jar

    +

    javax.ws.rs-api-2.1.1.jar

    +

    opentelemetry-semconv-1.16.0-alpha.jar

    +

    commons-collections-3.2.2.jar

    +

    javolution-5.5.1.jar

    +

    opentracing-api-0.33.0.jar

    +

    commons-compiler-3.0.16.jar

    +

    jaxb-api-2.2.11.jar

    +

    opentracing-noop-0.33.0.jar

    +

    commons-compress-1.21.jar

    +

    jaxb-runtime-2.3.2.jar

    +

    opentracing-tracerresolver-0.1.8.jar

    +

    commons-configuration2-2.1.1.jar

    +

    jboss-logging-3.4.1.Final.jar

    +

    opentracing-util-0.33.0.jar

    +

    commons-crypto-1.0.0-20191105.jar

    +

    jboss-threads-2.3.3.Final.jar

    +

    orc-core-1.6.8.jar

    +

    commons-daemon-1.0.13.jar

    +

    jcip-annotations-1.0-1.jar

    +

    orc-mapreduce-1.6.8.jar

    +

    commons-dbcp-1.4.jar

    +

    jcl-over-slf4j-1.7.36.jar

    +

    orc-shims-1.6.8.jar

    +

    commons-dbcp2-2.6.0.jar

    +

    jcodings-1.0.57.jar

    +

    orc-tools-1.6.7-h0.cbu.mrs.321.r10.jar

    +

    commons-digester-2.1.jar

    +

    jdo-api-3.2.jar

    +

    oro-2.0.8.jar

    +

    commons-httpclient-3.1.jar

    +

    jersey-client-2.34.jar

    +

    osgi-resource-locator-1.0.3.jar

    +

    commons-io-2.8.0.jar

    +

    jersey-common-2.34.jar

    +

    paranamer-2.8.jar

    +

    commons-lang-2.4.jar

    +

    jersey-container-servlet-2.34.jar

    +

    parquet-column-1.12.2.jar

    +

    commons-lang-2.6.jar

    +

    jersey-container-servlet-core-2.34.jar

    +

    parquet-common-1.12.2.jar

    +

    commons-lang3-3.10.jar

    +

    jersey-hk2-2.34.jar

    +

    parquet-encoding-1.12.2.jar

    +

    commons-logging-1.2.jar

    +

    jersey-server-2.34.jar

    +

    parquet-format-structures-1.12.2.jar

    +

    commons-math3-3.4.1.jar

    +

    jets3t-0.9.4-1.0.0.jar

    +

    parquet-hadoop-1.12.2.jar

    +

    commons-net-3.1.jar

    +

    jettison-1.1.jar

    +

    parquet-hadoop-bundle-1.12.0-ei-2.0.jar

    +

    commons-pool2-2.6.1.jar

    +

    jetty-http-9.4.41.v20210516.jar

    +

    parquet-jackson-1.12.2.jar

    +

    commons-text-1.10.0.jar

    +

    jetty-io-9.4.41.v20210516.jar

    +

    postgresql-42.3.5.jar

    +

    commons-validator-1.7.jar

    +

    jetty-rewrite-9.4.43.v20210629.jar

    +

    protobuf-java-2.5.0.jar

    +

    compress-lzf-1.0.3.jar

    +

    jetty-security-9.4.41.v20210516.jar

    +

    py4j-0.10.9.jar

    +

    core-1.1.2.jar

    +

    jetty-server-9.4.41.v20210516.jar

    +

    pyrolite-4.30.jar

    +

    curator-client-2.13.0.jar

    +

    jetty-servlet-9.4.41.v20210516.jar

    +

    re2j-1.1.jar

    +

    curator-framework-2.13.0.jar

    +

    jetty-util-9.4.41.v20210516.jar

    +

    RoaringBitmap-0.9.0.jar

    +

    curator-recipes-2.13.0.jar

    +

    jetty-util-ajax-9.4.41.v20210516.jar

    +

    scala-collection-compat_2.12-2.1.1.jar

    +

    datanucleus-api-jdo-4.2.4.jar

    +

    jetty-webapp-9.4.41.v20210516.jar

    +

    scala-compiler-2.12.16.jar

    +

    datanucleus-core-4.1.17.jar

    +

    jetty-xml-9.4.41.v20210516.jar

    +

    scala-library-2.12.16.jar

    +

    datanucleus-rdbms-fi-4.1.19-302022.jar

    +

    JLargeArrays-1.5.jar

    +

    scala-parser-combinators_2.12-1.1.2.jar

    +

    derby-10.14.2.0.jar

    +

    jline-3.21.0.jar

    +

    scala-reflect-2.12.16.jar

    +

    disruptor-3.4.2.jar

    +

    joda-time-2.10.5.jar

    +

    scala-xml_2.12-1.2.0.jar

    +

    dli-catalog-client-2.3.7-20240108.090504-101.jar

    +

    jodd-core-3.5.2.jar

    +

    secComponentApi-1.1.8.jar

    +

    dli-catalog-hive3-client-2.3.7-20240108.090513-100.jar

    +

    jodd-util-6.0.0.jar

    +

    serializer-2.7.2.jar

    +

    dli-catalog-hive-extension-2.3.7-20240108.090517-100.jar

    +

    joni-2.1.43.jar

    +

    shapeless_2.12-2.3.3.jar

    +

    dnsjava-2.1.7.jar

    +

    jpam-1.1.jar

    +

    shims-0.9.0.jar

    +

    dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar

    +

    jsch-0.1.72.jar

    +

    sketches-core-0.9.0.jar

    +

    error_prone_annotations-2.18.0.jar

    +

    json-20210307.jar

    +

    slf4j-api-1.7.30.jar

    +

    esdk-obs-java-optimised-3.22.10.2.jar

    +

    json4s-ast_2.12-3.7.0-M5.jar

    +

    slf4j-log4j12-1.7.25.jar

    +

    esri-geometry-api-2.2.0.jar

    +

    json4s-core_2.12-3.7.0-M5.jar

    +

    snakeyaml-1.30.jar

    +

    fastutil-6.5.6.jar

    +

    json4s-jackson_2.12-3.7.0-M5.jar

    +

    snappy-java-1.1.8.2.jar

    +

    flatbuffers-java-1.9.0.jar

    +

    json4s-scalap_2.12-3.7.0-M5.jar

    +

    spark-avro_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    generex-1.0.2.jar

    +

    json-path-2.4.0.jar

    +

    spark-catalyst_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    glassfish-corba-omgapi-4.2.2.jar

    +

    json-smart-2.3.jar

    +

    spark-core_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    gson-2.8.9.jar

    +

    jsr305-3.0.0.jar

    +

    spark-graphx_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    gson-fire-1.8.5.jar

    +

    JTransforms-3.1.jar

    +

    spark-hive_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    guava-14.0.1.jar

    +

    jul-to-slf4j-1.7.36.jar

    +

    spark-kubernetes_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    guice-3.0.jar

    +

    kafka-clients-2.8.0.jar

    +

    spark-kvstore_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    guice-assistedinject-3.0.jar

    +

    kerb-admin-2.0.2.jar

    +

    spark-launcher_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    guice-servlet-4.0.jar

    +

    kerb-client-2.0.2.jar

    +

    spark-mllib_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-annotations-3.1.1-h0.cbu.mrs.313.r9.jar

    +

    kerb-common-2.0.2.jar

    +

    spark-mllib-local_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-archives-3.3.1-h0.cbu.mrs.321.r10.jar

    +

    kerb-core-2.0.2.jar

    +

    spark-network-common_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-auth-3.3.1-h0.cbu.mrs.321.r16.jar

    +

    kerb-crypto-2.0.2.jar

    +

    spark-network-shuffle_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-3.3.1-h0.cbu.mrs.321.r16.jar

    +

    kerb-identity-2.0.2.jar

    +

    spark-quota-manager_2.12-3.1.1-2.3.7.dli-SNAPSHOT.jar

    +

    hadoop-client-3.1.1-h0.cbu.mrs.313.r9.jar

    +

    kerb-server-2.0.2.jar

    +

    spark-repl_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-common-3.3.1-h0.cbu.mrs.321.r10.jar

    +

    kerb-simplekdc-2.0.2.jar

    +

    spark-sketch_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-distcp-3.3.1-h0.cbu.mrs.321.r10.jar

    +

    kerb-util-2.0.2.jar

    +

    spark-sql_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-hdfs-3.3.1-h0.cbu.mrs.321.r16.jar

    +

    kerby-asn1-2.0.2.jar

    +

    spark-streaming_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-hdfs-client-3.3.1-h0.cbu.mrs.321.r10.jar

    +

    kerby-config-2.0.2.jar

    +

    spark-tags_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-3.1.1-52.1.jar

    +

    kerby-pkix-2.0.2.jar

    +

    spark-unsafe_2.12-3.1.1-h1.cbu.dli.20230607.r1.jar

    +

    hadoop-mapreduce-client-common-3.1.1-h0.cbu.mrs.313.r9.jar

    +

    kerby-util-2.0.2.jar

    +

    spark-uquery_2.12-3.1.1-2.3.7.dli-SNAPSHOT.jar

    +

    hadoop-mapreduce-client-core-3.1.1-h0.cbu.mrs.313.r9.jar

    +

    kerby-xdr-2.0.2.jar

    +

    spire_2.12-0.17.0-M1.jar

    +

    hadoop-mapreduce-client-jobclient-3.1.1-h0.cbu.mrs.313.r9.jar

    +

    kotlin-stdlib-1.4.21.jar

    +

    spire-macros_2.12-0.17.0-M1.jar

    +

    hadoop-mapreduce-client-nativetask-3.3.1-h0.cbu.mrs.321.r10.jar

    +

    kotlin-stdlib-common-1.4.21.jar

    +

    spire-platform_2.12-0.17.0-M1.jar

    +

    hadoop-registry-3.3.1-h0.cbu.mrs.321.r10.jar

    +

    kryo-shaded-4.0.2.jar

    +

    spire-util_2.12-0.17.0-M1.jar

    +

    hadoop-shaded-guava-1.1.1.jar

    +

    kubernetes-client-5.4.1-20211025.jar

    +

    sqlline-1.3.0.jar

    +

    hadoop-shaded-protobuf_3_7-1.1.1.jar

    +

    kubernetes-model-admissionregistration-5.4.1-20211025.jar

    +

    ST4-4.0.4.jar

    +

    hadoop-yarn-api-3.1.1-h0.cbu.mrs.313.r9.jar

    +

    kubernetes-model-apiextensions-5.4.1-20211025.jar

    +

    stax2-api-4.2.1.jar

    +

    hadoop-yarn-client-3.1.1-h0.cbu.mrs.313.r9.jar

    +

    kubernetes-model-apps-5.4.1-20211025.jar

    +

    stax-api-1.0.1.jar

    +

    hadoop-yarn-registry-3.3.1-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-autoscaling-5.4.1-20211025.jar

    +

    stream-2.9.6.jar

    +

    hbase-asyncfs-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-batch-5.4.1-20211025.jar

    +

    streamingClient

    +

    hbase-client-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-certificates-5.4.1-20211025.jar

    +

    streamingClient010

    +

    hbase-common-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-common-5.4.1-20211025.jar

    +

    swagger-annotations-2.2.8.jar

    +

    hbase-hadoop2-compat-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-coordination-5.4.1-20211025.jar

    +

    tephra-api-0.6.0.jar

    +

    hbase-hadoop-compat-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-core-5.4.1-20211025.jar

    +

    tephra-core-0.6.0.jar

    +

    hbase-http-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-discovery-5.4.1-20211025.jar

    +

    tephra-hbase-compat-1.0-0.6.0.jar

    +

    hbase-logging-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-events-5.4.1-20211025.jar

    +

    threetenbp-1.3.5.jar

    +

    hbase-metrics-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-extensions-5.4.1-20211025.jar

    +

    threeten-extra-1.5.0.jar

    +

    hbase-metrics-api-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-flowcontrol-5.4.1-20211025.jar

    +

    tink-1.6.0.jar

    +

    hbase-procedure-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-metrics-5.4.1-20211025.jar

    +

    token-provider-2.0.2.jar

    +

    hbase-protocol-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-networking-5.4.1-20211025.jar

    +

    tomcat-servlet-api-8.5.61.jar

    +

    hbase-protocol-shaded-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-node-5.4.1-20211025.jar

    +

    transaction-api-1.1.jar

    +

    hbase-replication-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-policy-5.4.1-20211025.jar

    +

    twill-api-0.6.0-incubating.jar

    +

    hbase-server-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    kubernetes-model-rbac-5.4.1-20211025.jar

    +

    twill-common-0.6.0-incubating.jar

    +

    hbase-shaded-gson-4.1.4.jar

    +

    kubernetes-model-scheduling-5.4.1-20211025.jar

    +

    twill-core-0.6.0-incubating.jar

    +

    hbase-shaded-jersey-4.1.4.jar

    +

    kubernetes-model-storageclass-5.4.1-20211025.jar

    +

    twill-discovery-api-0.6.0-incubating.jar

    +

    hbase-shaded-jetty-4.1.4.jar

    +

    leveldbjni-all-1.8-20191105.jar

    +

    twill-discovery-core-0.6.0-incubating.jar

    +

    hbase-shaded-miscellaneous-4.1.4.jar

    +

    libfb303-0.9.3.jar

    +

    twill-zookeeper-0.6.0-incubating.jar

    +

    hbase-shaded-netty-4.1.4.jar

    +

    libthrift-0.14.1-ei-311001.jar

    +

    univocity-parsers-2.9.1.jar

    +

    hbase-shaded-protobuf-4.1.4.jar

    +

    log4j-1.2.17-cloudera1.jar

    +

    us-common-1.0.66.jar

    +

    hbase-unsafe-4.1.4.jar

    +

    log4j-api-2.17.1.jar

    +

    velocity-1.7.jar

    +

    hbase-zookeeper-2.4.14-h0.cbu.mrs.321.r10.jar

    +

    log4j-rolling-appender-20131024-2017.jar

    +

    velocity-engine-core-2.3.jar

    +

    hibernate-validator-6.2.5.Final.jar

    +

    logging-interceptor-3.14.9.jar

    +

    wildfly-client-config-1.0.1.Final.jar

    +

    HikariCP-2.6.1.jar

    +

    luxor-cluster-quota-manager-transport_2.12-2.3.7-20231226.034700-559.jar

    +

    wildfly-common-1.5.2.Final.jar

    +

    hive-classification-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    luxor-encrypt-2.3.7-20231226.034423-1046.jar

    +

    woodstox-core-5.4.0.jar

    +

    hive-common-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    luxor-fs3-2.3.7-20231226.034438-1039.jar

    +

    xalan-2.7.2.jar

    +

    hive-exec-3.1.0-h0.cbu.mrs.321.r10-core.jar

    +

    luxor-obs-fs3-2.3.7-20231226.034443-1038.jar

    +

    xbean-asm7-shaded-4.15.jar

    +

    hive-llap-client-2.3.3-ei-12-20210120.005053-2.jar

    +

    luxor-rpc_2.12-2.3.7-20231226.034653-560.jar

    +

    xercesImpl-2.12.2.jar

    +

    hive-llap-common-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    luxor-scc-adapter-2.3.7-20231226.034418-1045.jar

    +

    xml-apis-1.4.01.jar

    +

    hive-llap-tez-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    luxor-transport-2.3.7-20231226.034433-1038.jar

    +

    xnio-api-3.8.4.Final.jar

    +

    hive-metastore-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    lz4-java-1.7.1.jar

    +

    xz-1.5.jar

    +

    hive-serde-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    machinist_2.12-0.6.8.jar

    +

    zjsonpatch-0.3.0.jar

    +

    hive-service-rpc-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    macro-compat_2.12-1.1.1.jar

    +

    zookeeper-3.5.6-ei-302002.jar

    +

    hive-shims-0.23-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    memarts-ccsdk-1.0.jar

    +

    zookeeper-jute-3.5.6-ei-302002.jar

    +

    hive-shims-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    memory-0.9.0.jar

    +

    zstd-jni-1.4.9-1.jar

    +

    hive-shims-common-3.1.0-h0.cbu.mrs.321.r10.jar

    +

    metrics-core-4.1.1.jar

    +

    -

    +
    -

    Spark 2.3.2 Dependencies

    • accessors-smart-1.2.jar
    • activation-1.1.1.jar
    • aircompressor-0.8.jar
    • alluxio-2.3.1-luxor-SNAPSHOT-client.jar
    • antlr-2.7.7.jar
    • antlr4-runtime-4.8-1.jar
    • antlr-runtime-3.4.jar
    • aopalliance-1.0.jar
    • aopalliance-repackaged-2.4.0-b34.jar
    • apache-log4j-extras-1.2.17.jar
    • arpack_combined_all-0.1.jar
    • arrow-format-0.8.0.jar
    • arrow-memory-0.8.0.jar
    • arrow-vector-0.8.0.jar
    • asm-5.0.4.jar
    • audience-annotations-0.5.0.jar
    • automaton-1.11-8.jar
    • avro-1.7.7.jar
    • avro-ipc-1.7.7.jar
    • avro-ipc-1.7.7-tests.jar
    • avro-mapred-1.7.7-hadoop2.jar
    • java-sdk-bundle-1.11.271.jar
    • base64-2.3.8.jar
    • bcpkix-jdk15on-1.66.jar
    • bcprov-jdk15on-1.66.jar
    • bonecp-0.8.0.RELEASE.jar
    • breeze_2.11-0.13.2.jar
    • breeze-macros_2.11-0.13.2.jar
    • calcite-avatica-1.2.0-incubating.jar
    • calcite-core-1.2.0-incubating.jar
    • calcite-linq4j-1.2.0-incubating.jar
    • checker-qual-2.11.1.jar
    • chill_2.11-0.8.4.jar
    • chill-java-0.8.4.jar
    • commons-beanutils-1.9.4.jar
    • commons-cli-1.2.jar
    • commons-codec-2.0-20130428.202122-59.jar
    • commons-collections-3.2.2.jar
    • commons-collections4-4.2.jar
    • commons-compiler-3.0.8.jar
    • commons-compress-1.4.1.jar
    • commons-configuration2-2.1.1.jar
    • commons-crypto-1.0.0-20191105.jar
    • commons-daemon-1.0.13.jar
    • commons-dbcp-1.4.jar
    • commons-dbcp2-2.7.0.jar
    • commons-httpclient-3.1.jar
    • commons-io-2.5.jar
    • commons-lang-2.6.jar
    • commons-lang3-3.5.jar
    • commons-logging-1.2.jar
    • commons-math3-3.4.1.jar
    • commons-net-2.2.jar
    • commons-pool-1.5.4.jar
    • commons-pool2-2.8.0.jar
    • commons-text-1.3.jar
    • compress-lzf-1.0.3.jar
    • core-1.1.2.jar
    • curator-client-4.2.0.jar
    • curator-framework-4.2.0.jar
    • curator-recipes-2.7.1.jar
    • datanucleus-api-jdo-3.2.6.jar
    • datanucleus-core-3.2.10.jar
    • datanucleus-rdbms-3.2.9.jar
    • derby-10.12.1.1.jar
    • dnsjava-2.1.7.jar
    • ehcache-3.3.1.jar
    • eigenbase-properties-1.1.5.jar
    • error_prone_annotations-2.3.4.jar
    • failureaccess-1.0.1.jar
    • fastutil-8.2.3.jar
    • ffmpeg-4.3.1-1.5.4.jar
    • ffmpeg-4.3.1-1.5.4-linux-x86_64.jar
    • flatbuffers-1.2.0-3f79e055.jar
    • generex-1.0.2.jar
    • geronimo-jcache_1.0_spec-1.0-alpha-1.jar
    • gson-2.2.4.jar
    • guava-29.0-jre.jar
    • guice-4.0.jar
    • guice-servlet-4.0.jar
    • hadoop-annotations-3.1.1-ei-302002.jar
    • hadoop-auth-3.1.1-ei-302002.jar
    • hadoop-3.1.1-ei-302002.jar
    • hadoop-client-3.1.1-ei-302002.jar
    • hadoop-common-3.1.1-ei-302002.jar
    • hadoop-hdfs-3.1.1-ei-302002.jar
    • hadoop-hdfs-client-3.1.1-ei-302002.jar
    • hadoop-mapreduce-client-common-3.1.1-ei-302002.jar
    • hadoop-mapreduce-client-core-3.1.1-ei-302002.jar
    • hadoop-mapreduce-client-jobclient-3.1.1-ei-302002.jar
    • hadoop-minikdc-3.1.1-ei-302002.jar
    • hadoop-yarn-api-3.1.1-ei-302002.jar
    • hadoop-yarn-client-3.1.1-ei-302002.jar
    • hadoop-yarn-common-3.1.1-ei-302002.jar
    • hadoop-yarn-registry-3.1.1-ei-302002.jar
    • hadoop-yarn-server-common-3.1.1-ei-302002.jar
    • hadoop-yarn-server-web-proxy-3.1.1-ei-302002.jar
    • hamcrest-core-1.3.jar
    • HikariCP-java7-2.4.12.jar
    • hive-common-1.2.1-2.1.0.dli-20201111.064115-91.jar
    • hive-exec-1.2.1-2.1.0.dli-20201111.064444-91.jar
    • hive-metastore-1.2.1-2.1.0.dli-20201111.064230-91.jar
    • hk2-api-2.4.0-b34.jar
    • hk2-locator-2.4.0-b34.jar
    • hk2-utils-2.4.0-b34.jar
    • hppc-0.7.2.jar
    • htrace-core4-4.2.0-incubating-1.0.0.jar
    • httpclient-4.5.4.jar
    • httpcore-4.4.7.jar
    • ivy-2.4.0.jar
    • j2objc-annotations-1.3.jar
    • jackson-annotations-2.10.0.jar
    • jackson-core-2.10.0.jar
    • jackson-core-asl-1.9.13-atlassian-4.jar
    • jackson-databind-2.10.0.jar
    • jackson-dataformat-yaml-2.10.0.jar
    • jackson-datatype-jsr310-2.10.3.jar
    • jackson-jaxrs-base-2.10.3.jar
    • jackson-jaxrs-json-provider-2.10.3.jar
    • jackson-mapper-asl-1.9.13-atlassian-4.jar
    • jackson-module-jaxb-annotations-2.10.3.jar
    • jackson-module-paranamer-2.10.0.jar
    • jackson-module-scala_2.11-2.10.0.jar
    • jakarta.activation-api-1.2.1.jar
    • jakarta.xml.bind-api-2.3.2.jar
    • janino-3.0.8.jar
    • javacpp-1.5.4.jar
    • javacpp-1.5.4-linux-x86_64.jar
    • javacv-1.5.4.jar
    • JavaEWAH-1.1.7.jar
    • javassist-3.18.1-GA.jar
    • javax.annotation-api-1.2.jar
    • javax.inject-1.jar
    • javax.inject-2.4.0-b34.jar
    • javax.servlet-api-3.1.0.jar
    • javax.ws.rs-api-2.0.1.jar
    • java-xmlbuilder-1.1.jar
    • javolution-5.3.1.jar
    • jaxb-api-2.2.11.jar
    • jcip-annotations-1.0-1.jar
    • jcl-over-slf4j-1.7.26.jar
    • jdo-api-3.0.1.jar
    • jersey-client-2.23.1.jar
    • jersey-common-2.23.1.jar
    • jersey-container-servlet-2.23.1.jar
    • jersey-container-servlet-core-2.23.1.jar
    • jersey-guava-2.23.1.jar
    • jersey-media-jaxb-2.23.1.jar
    • jersey-server-2.23.1.jar
    • jets3t-0.9.4.jar
    • jetty-http-9.4.31.v20200723.jar
    • jetty-io-9.4.31.v20200723.jar
    • jetty-security-9.4.31.v20200723.jar
    • jetty-server-9.4.31.v20200723.jar
    • jetty-servlet-9.4.31.v20200723.jar
    • jetty-util-9.4.31.v20200723.jar
    • jetty-util-ajax-9.4.31.v20200723.jar
    • jetty-webapp-9.4.31.v20200723.jar
    • jetty-xml-9.4.31.v20200723.jar
    • joda-time-2.9.3.jar
    • jodd-core-4.2.0.jar
    • json-20200518.jar
    • json4s-ast_2.11-3.2.11.jar
    • json4s-core_2.11-3.2.11.jar
    • json4s-jackson_2.11-3.2.11.jar
    • json-sanitizer-1.2.1.jar
    • json-smart-2.3.jar
    • jsp-api-2.1.jar
    • jsr305-3.0.2.jar
    • jta-1.1.jar
    • jtransforms-2.4.0.jar
    • jul-to-slf4j-1.7.26.jar
    • junit-4.11.jar
    • kerb-admin-1.0.1.jar
    • kerb-client-1.0.1.jar
    • kerb-common-1.0.1.jar
    • kerb-core-1.0.1.jar
    • kerb-crypto-1.0.1.jar
    • kerb-identity-1.0.1.jar
    • kerb-server-1.0.1.jar
    • kerb-simplekdc-1.0.1.jar
    • kerb-util-1.0.1.jar
    • kerby-asn1-1.0.1.jar
    • kerby-config-1.0.1.jar
    • kerby-pkix-1.0.1.jar
    • kerby-util-1.0.1.jar
    • kerby-xdr-1.0.1.jar
    • kryo-shaded-3.0.3.jar
    • kubernetes-client-4.9.2-20200804.jar
    • kubernetes-model-4.9.2-20200804.jar
    • kubernetes-model-common-4.9.2-20200804.jar
    • leveldbjni-all-1.8-20191105.jar
    • libfb303-0.9.3.jar
    • libthrift-0.12.0.jar
    • listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
    • log4j-1.2.17-cloudera1.jar
    • log4j-rolling-appender-20131024-2017.jar
    • logging-interceptor-3.14.4.jar
    • luxor-encrypt-2.1.0-20201106.065437-53.jar
    • luxor-fs3-2.1.0-20201106.065612-53.jar
    • luxor-obs-fs3-2.1.0-20201106.065616-53.jar
    • luxor-rpc_2.11-2.1.0-20201106.065541-53.jar
    • luxor-rpc-protobuf2-2.1.0-20201106.065551-53.jar
    • lz4-java-1.7.1.jar
    • machinist_2.11-0.6.1.jar
    • macro-compat_2.11-1.1.1.jar
    • metrics-core-3.1.5.jar
    • metrics-graphite-3.1.5.jar
    • metrics-jmx-4.1.12.1.jar
    • metrics-json-3.1.5.jar
    • metrics-jvm-3.1.5.jar
    • minlog-1.3.0.jar
    • mssql-jdbc-6.2.1.jre7.jar
    • netty-3.10.6.Final.jar
    • netty-all-4.1.51.Final.jar
    • nimbus-jose-jwt-8.19.jar
    • objenesis-2.1.jar
    • okhttp-3.14.4.jar
    • okio-1.17.2.jar
    • opencsv-2.3.jar
    • opencsv-4.6.jar
    • opencv-4.3.0-2.jar
    • orc-core-1.4.4-nohive.jar
    • orc-mapreduce-1.4.4-nohive.jar
    • oro-2.0.8.jar
    • osgi-resource-locator-1.0.1.jar
    • paranamer-2.8.jar
    • parquet-column-1.8.3.jar
    • parquet-common-1.8.3.jar
    • parquet-encoding-1.8.3.jar
    • parquet-format-2.3.1.jar
    • parquet-hadoop-1.8.3.jar
    • parquet-hadoop-bundle-1.6.0.jar
    • parquet-jackson-1.8.3.jar
    • parquet-format-2.3.1.jar
    • parquet-hadoop-1.8.3.jar
    • parquet-hadoop-bundle-1.6.0.jar
    • parquet-jackson-1.8.3.jar
    • postgresql-42.2.14.jar
    • protobuf-java-2.5.0.jar
    • py4j-0.10.7.jar
    • pyrolite-4.13.jar
    • re2j-1.1.jar
    • RoaringBitmap-0.5.11.jar
    • scala-compiler-2.11.12.jar
    • scala-library-2.11.12.jar
    • scalap-2.11.0.jar
    • scala-parser-combinators_2.11-1.1.0.jar
    • scala-reflect-2.11.12.jar
    • scala-xml_2.11-1.0.5.jar
    • secComponentApi-1.0.5c.jar
    • shapeless_2.11-2.3.2.jar
    • slf4j-api-1.7.30.jar
    • slf4j-log4j12-1.7.30.jar
    • snakeyaml-1.24.jar
    • snappy-java-1.1.7.5.jar
    • spark-catalyst_2.11-2.3.2.0101-2.1.0.dli-20201111.073826-143.jar
    • spark-core_2.11-2.3.2.0101-.0.dli-20201111.073836-134.jar
    • spark-graphx_2.11-2.3.2.0101-2.1.0.dli-20201111.073847-129.jar
    • spark-hive_2.11-2.3.2.0101-.0.dli-20201111.073854-132.jar
    • spark-kubernetes_2.11-2.3.2.0101-2.1.0.dli-20201111.073916-85.jar
    • spark-kvstore_2.11-2.3.2.0101-2.1.0.dli-20201111.073933-127.jar
    • spark-launcher_2.11-2.3.2.0101-2.1.0.dli-20201111.073940-127.jar
    • spark-mllib_2.11-2.3.2.0101-2.1.0.dli-20201111.073946-127.jar
    • spark-mllib-local_2.11-2.3.2.0101-2.1.0.dli-20201111.073953-127.jar
    • spark-network-common_2.11-2.3.2.0101-2.1.0.dli-20201111.073959-127.jar
    • spark-network-shuffle_2.11-2.3.2.0101-2.1.0.dli-20201111.074007-127.jar
    • spark-om_2.11-2.3.2.0101-.0.dli-20201111.074019-125.jar
    • spark-repl_2.11-2.3.2.0101-2.1.0.dli-20201111.074028-125.jar
    • spark-sketch_2.11-2.3.2.0101-2.1.0.dli-20201111.074035-125.jar
    • spark-sql_2.11-2.3.2.0101-2.1.0.dli-20201111.074041-126.jar
    • spark-streaming_2.11-2.3.2.0101-2.1.0.dli-20201111.074100-123.jar
    • spark-tags_2.11-2.3.2.0101-2.1.0.dli-20201111.074136-123.jar
    • spark-tags_2.11-2.3.2.0101-2.1.0.dli-20201111.074141-124-tests.jar
    • spark-unsafe_2.11-2.3.2.0101-2.1.0.dli-20201111.074144-123.jar
    • spark-uquery_2.11-2.3.2.0101-2.1.0.dli-20201111.074906-210.jar
    • spark-yarn_2.11-2.3.2.0101-2.1.0.dli-20201111.074151-123.jar
    • spire_2.11-0.13.0.jar
    • spire-macros_2.11-0.13.0.jar
    • ST4-4.3.1.jar
    • stax2-api-3.1.4.jar
    • stax-api-1.0-2.jar
    • stream-2.7.0.jar
    • stringtemplate-3.2.1.jar
    • token-provider-1.0.1.jar
    • univocity-parsers-2.5.9.jar
    • validation-api-1.1.0.Final.jar
    • woodstox-core-5.0.3.jar
    • xbean-asm5-shaded-4.4.jar
    • xercesImpl-2.12.0.jar
    • xml-apis-1.4.01.jar
    • xz-1.0.jar
    • zjsonpatch-0.3.0.jar
    • zookeeper-3.5.6-ei-302002.jar
    • zookeeper-jute-3.5.6-ei-302002.jar
    • zstd-jni-1.4.4-11.jar
    -

    Flink 1.7.2 Dependencies

    • bcpkix-jdk15on-1.60.jar
    • bcprov-jdk15on-1.60.jar
    • commons-codec-1.9.jar
    • commons-configuration-1.7.jar
    • deeplearning4j-core-0.9.1.jar
    • deeplearning4j-nlp-0.9.1.jar
    • deeplearning4j-nn-0.9.1.jar
    • ejml-cdense-0.33.jar
    • ejml-core-0.33.jar
    • ejml-ddense-0.33.jar
    • ejml-dsparse-0.33.jar
    • ejml-experimental-0.33.jar
    • ejml-fdense-0.33.jar
    • ejml-simple-0.33.jar
    • ejml-zdense-0.33.jar
    • elsa-3.0.0-M7.jar
    • esdk-obs-java-3.1.3.jar
    • flink-cep_2.11-1.7.0.jar
    • flink-cep-scala_2.11-1.7.0.jar
    • flink-dist_2.11-1.7.0.jar
    • flink-gelly_2.11-1.7.0.jar
    • flink-gelly-scala_2.11-1.7.0.jar
    • flink-ml_2.11-1.7.0.jar
    • flink-python_2.11-1.7.0.jar
    • flink-queryable-state-runtime_2.11-1.7.0.jar
    • flink-shaded-curator-1.7.0.jar
    • flink-shaded-hadoop2-uber-1.7.0.jar
    • flink-table_2.11-1.7.0.jar
    • guava-26.0-jre.jar
    • hadoop-3.1.1-41-20201014.085840-4.jar
    • httpasyncclient-4.1.2.jar
    • httpclient-4.5.12.jar
    • httpcore-4.4.4.jar
    • httpcore-nio-4.4.4.jar
    • java-xmlbuilder-1.1.jar
    • jna-4.1.0.jar
    • libtensorflow-1.12.0.jar
    • log4j-api-2.8.2.jar
    • log4j-core-2.8.2.jar
    • log4j-over-slf4j-1.7.21.jar
    • logback-classic-1.2.3.jar
    • logback-core-1.2.3.jar
    • nd4j-api-0.9.1.jar
    • nd4j-native-0.9.1.jar
    • nd4j-native-api-0.9.1.jar
    • nd4j-native-platform-0.9.1.jar
    • okhttp-3.14.8.jar
    • okio-1.14.0.jar
    • slf4j-api-1.7.21.jar
    • tensorflow-1.12.0.jar
    +

    Spark 2.4.5 Dependencies

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 2 Spark 2.4.5 dependencies

    Dependency

    +

    JavaEWAH-1.1.7.jar

    +

    httpclient-4.5.6.jar

    +

    lucene-queryparser-7.7.2.jar

    +

    RoaringBitmap-0.7.45.jar

    +

    httpcore-4.4.10.jar

    +

    lucene-sandbox-7.7.2.jar

    +

    ST4-4.3.1.jar

    +

    ivy-2.4.0.jar

    +

    luxor-encrypt-2.0.0-20220623.010726-213.jar

    +

    accessors-smart-1.2.jar

    +

    jackson-annotations-2.11.4.jar

    +

    luxor-fs3-2.0.0-20220623.010750-209.jar

    +

    activation-1.1.1.jar

    +

    jackson-core-2.11.4.jar

    +

    luxor-obs-fs3-2.0.0-20220623.010756-209.jar

    +

    aircompressor-0.16.jar

    +

    jackson-core-asl-1.9.13-atlassian-4.jar

    +

    luxor-rpc_2.11-2.0.0-20220623.010737-182.jar

    +

    alluxio-2.3.1-luxor-SNAPSHOT-client.jar

    +

    jackson-databind-2.11.4.jar

    +

    luxor-transport-2.0.0-20220623.010744-71.jar

    +

    annotations-17.0.0.jar

    +

    jackson-dataformat-yaml-2.11.4.jar

    +

    lz4-java-1.7.1.jar

    +

    antlr-2.7.7.jar

    +

    jackson-datatype-jsr310-2.11.2.jar

    +

    machinist_2.11-0.6.1.jar

    +

    antlr-runtime-3.4.jar

    +

    jackson-jaxrs-base-2.10.3.jar

    +

    macro-compat_2.11-1.1.1.jar

    +

    antlr4-runtime-4.8-1.jar

    +

    jackson-jaxrs-json-provider-2.10.3.jar

    +

    metrics-core-3.1.5.jar

    +

    aopalliance-1.0.jar

    +

    jackson-mapper-asl-1.9.13-atlassian-4.jar

    +

    metrics-graphite-3.1.5.jar

    +

    aopalliance-repackaged-2.4.0-b34.jar

    +

    jackson-module-jaxb-annotations-2.10.3.jar

    +

    metrics-jmx-4.1.12.1.jar

    +

    apache-log4j-extras-1.2.17.jar

    +

    jackson-module-paranamer-2.11.4.jar

    +

    metrics-json-3.1.5.jar

    +

    arpack_combined_all-0.1.jar

    +

    jackson-module-scala_2.11-2.11.4.jar

    +

    metrics-jvm-3.1.5.jar

    +

    arrow-format-0.12.0.jar

    +

    jakarta.activation-api-1.2.1.jar

    +

    minlog-1.3.0.jar

    +

    arrow-memory-0.12.0.jar

    +

    jakarta.xml.bind-api-2.3.2.jar

    +

    mssql-jdbc-6.2.1.jre7.jar

    +

    arrow-vector-0.12.0.jar

    +

    janino-3.0.9.jar

    +

    netty-all-4.1.51.Final.jar

    +

    asm-5.0.4.jar

    +

    java-util-1.9.0.jar

    +

    nimbus-jose-jwt-8.19.jar

    +

    audience-annotations-0.5.0.jar

    +

    java-xmlbuilder-1.1.jar

    +

    objenesis-2.5.1.jar

    +

    automaton-1.11-8.jar

    +

    javassist-3.18.1-GA.jar

    +

    okhttp-3.14.9.jar

    +

    avro-1.8.2.jar

    +

    javax.annotation-api-1.2.jar

    +

    okio-1.17.2.jar

    +

    avro-ipc-1.8.2.jar

    +

    javax.inject-1.jar

    +

    opencsv-2.3.jar

    +

    avro-mapred-1.8.2.jar

    +

    javax.inject-2.4.0-b34.jar

    +

    opencsv-4.6.jar

    +

    java-sdk-bundle-1.11.856.jar

    +

    javax.servlet-api-3.1.0.jar

    +

    opencv-4.3.0-2.jar

    +

    base64-2.3.8.jar

    +

    javax.ws.rs-api-2.0.1.jar

    +

    orc-core-1.6.8-nohive.jar

    +

    bcpkix-jdk15on-1.66.jar

    +

    javolution-5.3.1.jar

    +

    orc-mapreduce-1.6.8-nohive.jar

    +

    bcprov-jdk15on-1.67.jar

    +

    jaxb-api-2.2.11.jar

    +

    orc-shims-1.6.8.jar

    +

    bonecp-0.8.0.RELEASE.jar

    +

    jcip-annotations-1.0-1.jar

    +

    oro-2.0.8.jar

    +

    breeze-macros_2.11-0.13.2.jar

    +

    jcl-over-slf4j-1.7.30.jar

    +

    osgi-resource-locator-1.0.1.jar

    +

    breeze_2.11-0.13.2.jar

    +

    jdo-api-3.0.1.jar

    +

    paranamer-2.8.jar

    +

    calcite-avatica-1.2.0-incubating.jar

    +

    jersey-client-2.23.1.jar

    +

    parquet-column-1.12.2.jar

    +

    chill-java-0.9.3.jar

    +

    jersey-common-2.23.1.jar

    +

    parquet-common-1.12.2.jar

    +

    chill_2.11-0.9.3.jar

    +

    jersey-container-servlet-2.23.1.jar

    +

    parquet-encoding-1.12.2.jar

    +

    commons-beanutils-1.9.4.jar

    +

    jersey-container-servlet-core-2.23.1.jar

    +

    parquet-format-structures-1.12.2.jar

    +

    commons-cli-1.2.jar

    +

    jersey-guava-2.23.1.jar

    +

    parquet-hadoop-1.12.2.jar

    +

    commons-codec-1.15.jar

    +

    jersey-media-jaxb-2.23.1.jar

    +

    parquet-hadoop-bundle-1.6.0.jar

    +

    commons-collections-3.2.2.jar

    +

    jersey-server-2.23.1.jar

    +

    parquet-jackson-1.12.2.jar

    +

    commons-collections4-4.2.jar

    +

    jets3t-0.9.4.jar

    +

    postgresql-42.2.14.jar

    +

    commons-compiler-3.0.9.jar

    +

    jettison-1.1.jar

    +

    protobuf-java-2.5.0.jar

    +

    commons-compress-1.4.1.jar

    +

    jetty-http-9.4.34.v20201102.jar

    +

    py4j-0.10.7.jar

    +

    commons-configuration2-2.1.1.jar

    +

    jetty-io-9.4.34.v20201102.jar

    +

    pyrolite-4.13.jar

    +

    commons-crypto-1.0.0-20191105.jar

    +

    jetty-security-9.4.34.v20201102.jar

    +

    re2j-1.1.jar

    +

    commons-daemon-1.0.13.jar

    +

    jetty-server-9.4.34.v20201102.jar

    +

    scala-compiler-2.11.12.jar

    +

    commons-dbcp2-2.7.0.jar

    +

    jetty-servlet-9.4.34.v20201102.jar

    +

    scala-library-2.11.12.jar

    +

    commons-httpclient-3.1.jar

    +

    jetty-util-9.4.34.v20201102.jar

    +

    scala-parser-combinators_2.11-1.1.2.jar

    +

    commons-io-2.5.jar

    +

    jetty-util-ajax-9.4.34.v20201102.jar

    +

    scala-reflect-2.11.12.jar

    +

    commons-lang-2.6.jar

    +

    jetty-webapp-9.4.34.v20201102.jar

    +

    scala-xml_2.11-1.0.5.jar

    +

    commons-lang3-3.5.jar

    +

    jetty-xml-9.4.34.v20201102.jar

    +

    secComponentApi-1.0.6.jar

    +

    commons-logging-1.2.jar

    +

    joda-time-2.9.3.jar

    +

    shapeless_2.11-2.3.2.jar

    +

    commons-math3-3.4.1.jar

    +

    jodd-core-3.5.2.jar

    +

    shims-0.7.45.jar

    +

    commons-net-3.1.jar

    +

    json-20200518.jar

    +

    slf4j-api-1.7.30.jar

    +

    commons-pool2-2.8.0.jar

    +

    json-io-2.5.1.jar

    +

    slf4j-log4j12-1.7.30.jar

    +

    commons-text-1.3.jar

    +

    json-sanitizer-1.2.1.jar

    +

    snakeyaml-1.26.jar

    +

    compress-lzf-1.0.3.jar

    +

    json-smart-2.3.jar

    +

    snappy-java-1.1.8.2.jar

    +

    core-1.1.2.jar

    +

    json4s-ast_2.11-3.5.3.jar

    +

    solr-core-7.7.2.jar

    +

    crypter-0.0.6.jar

    +

    json4s-core_2.11-3.5.3.jar

    +

    solr-solrj-7.7.2.jar

    +

    curator-client-4.2.0.jar

    +

    json4s-jackson_2.11-3.5.3.jar

    +

    spark-avro_2.11-2.4.5.0100-2.0.0.dli-20220617.085536-9.jar

    +

    curator-framework-4.2.0.jar

    +

    json4s-scalap_2.11-3.5.3.jar

    +

    spark-avro_2.11-4.0.0.jar

    +

    curator-recipes-2.7.1.jar

    +

    jsp-api-2.1.jar

    +

    spark-catalyst_2.11-2.4.5.0100-2.0.0.dli-20220617.085405-16.jar

    +

    datanucleus-api-jdo-3.2.6.jar

    +

    jsr305-1.3.9.jar

    +

    spark-core_2.11-2.4.5.0100-2.0.0.dli-20220617.085327-16.jar

    +

    datanucleus-core-3.2.10.jar

    +

    jta-1.1.jar

    +

    spark-graphx_2.11-2.4.5.0100-.0.dli-20220617.085336-16.jar

    +

    datanucleus-rdbms-3.2.9.jar

    +

    jtransforms-2.4.0.jar

    +

    spark-hive_2.11-2.4.5.0100-2.0.0.dli-20220617.085423-16.jar

    +

    derby-10.14.2.0.jar

    +

    jts-core-1.16.1.jar

    +

    spark-kubernetes_2.11-2.4.5.0100-2.0.0.dli-20220617.085519-16.jar

    +

    dnsjava-2.1.7.jar

    +

    jul-to-slf4j-1.7.30.jar

    +

    spark-kvstore_2.11-2.4.5.0100-2.0.0.dli-20220617.085249-16.jar

    +

    ecj-3.21.0.jar

    +

    junit-4.11.jar

    +

    spark-launcher_2.11-2.4.5.0100-2.0.0.dli-20220617.085435-16.jar

    +

    ehcache-3.3.1.jar

    +

    kerb-admin-1.0.1.jar

    +

    spark-mllib-local_2.11-2.4.5.0100-2.0.0.dli-20220617.085349-16.jar

    +

    expiringmap-0.5.9.jar

    +

    kerb-client-1.0.1.jar

    +

    spark-mllib_2.11-2.4.5.0100-2.0.0.dli-20220617.085342-16.jar

    +

    fastutil-8.2.3.jar

    +

    kerb-common-1.0.1.jar

    +

    spark-network-common_2.11-2.4.5.0100-2.0.0.dli-20220617.085254-16.jar

    +

    flatbuffers-java-1.9.0.jar

    +

    kerb-core-1.0.1.jar

    +

    spark-network-shuffle_2.11-2.4.5.0100-2.0.0.dli-20220617.085300-16.jar

    +

    fst-2.50.jar

    +

    kerb-crypto-1.0.1.jar

    +

    spark-om_2.11-2.4.5.0100-2.0.0.dli-20220617.085316-16.jar

    +

    generex-1.0.2.jar

    +

    kerb-identity-1.0.1.jar

    +

    spark-repl_2.11-2.4.5.0100-2.0.0.dli-20220617.085430-16.jar

    +

    geronimo-jcache_1.0_spec-1.0-alpha-1.jar

    +

    kerb-server-1.0.1.jar

    +

    spark-sketch_2.11-2.4.5.0100-2.0.0.dli-20220617.085243-16.jar

    +

    gson-2.2.4.jar

    +

    kerb-simplekdc-1.0.1.jar

    +

    spark-sql_2.11-2.4.5.0100-2.0.0.dli-20220617.085414-16.jar

    +

    guava-14.0.1.jar

    +

    kerb-util-1.0.1.jar

    +

    spark-streaming_2.11-2.4.5.0100-.0.dli-20220617.085359-16.jar

    +

    guice-4.0.jar

    +

    kerby-asn1-1.0.1.jar

    +

    spark-tags_2.11-2.4.5.0100-2.0.0.dli-20220617.085322-16.jar

    +

    guice-servlet-4.0.jar

    +

    kerby-config-1.0.1.jar

    +

    spark-unsafe_2.11-2.4.5.0100-2.0.0.dli-20220617.085311-16.jar

    +

    hadoop-annotations-3.1.1-ei-302002.jar

    +

    kerby-pkix-1.0.1.jar

    +

    spark-uquery_2.11-2.4.5.0100-2.0.0.dli-SNAPSHOT.jar

    +

    hadoop-auth-3.1.1-ei-302002.jar

    +

    kerby-util-1.0.1.jar

    +

    spark-yarn_2.11-2.4.5.0100-2.0.0.dli-20220617.085531-16.jar

    +

    hadoop-3.1.1-ei-302002.jar

    +

    kerby-xdr-1.0.1.jar

    +

    spire-macros_2.11-0.13.0.jar

    +

    hadoop-client-3.1.1-ei-302002.jar

    +

    kryo-shaded-4.0.2.jar

    +

    spire_2.11-0.13.0.jar

    +

    hadoop-common-3.1.1-ei-302002.jar

    +

    kubernetes-client-5.4.1-20211025.jar

    +

    stax-api-1.0-2.jar

    +

    hadoop-hdfs-3.1.1-ei-302002.jar

    +

    kubernetes-model-admissionregistration-5.4.1-20211025.jar

    +

    stax2-api-3.1.4.jar

    +

    hadoop-hdfs-client-3.1.1-ei-302002.jar

    +

    kubernetes-model-apiextensions-5.4.1-20211025.jar

    +

    stream-2.7.0.jar

    +

    hadoop-3.1.1-46.jar

    +

    kubernetes-model-apps-5.4.1-20211025.jar

    +

    stringtemplate-3.2.1.jar

    +

    hadoop-mapreduce-client-common-3.1.1-ei-302002.jar

    +

    kubernetes-model-autoscaling-5.4.1-20211025.jar

    +

    threeten-extra-1.5.0.jar

    +

    hadoop-mapreduce-client-core-3.1.1-ei-302002.jar

    +

    kubernetes-model-batch-5.4.1-20211025.jar

    +

    tink-1.6.0.jar

    +

    hadoop-mapreduce-client-jobclient-3.1.1-ei-302002.jar

    +

    kubernetes-model-certificates-5.4.1-20211025.jar

    +

    token-provider-1.0.1.jar

    +

    hadoop-minikdc-3.1.1-ei-302002.jar

    +

    kubernetes-model-common-5.4.1-20211025.jar

    +

    tomcat-api-9.0.39.jar

    +

    hadoop-yarn-api-3.1.1-ei-302002.jar

    +

    kubernetes-model-coordination-5.4.1-20211025.jar

    +

    zookeeper-jute-3.5.6-ei-302002.jar

    +

    hadoop-yarn-client-3.1.1-ei-302002.jar

    +

    kubernetes-model-core-5.4.1-20211025.jar

    +

    tomcat-el-api-9.0.39.jar

    +

    hadoop-yarn-common-3.1.1-ei-302002.jar

    +

    kubernetes-model-discovery-5.4.1-20211025.jar

    +

    tomcat-jasper-9.0.39.jar

    +

    hadoop-yarn-registry-3.1.1-ei-302002.jar

    +

    kubernetes-model-events-5.4.1-20211025.jar

    +

    tomcat-jasper-el-9.0.39.jar

    +

    hadoop-yarn-server-applicationhistoryservice-3.1.1-ei-302002.jar

    +

    kubernetes-model-extensions-5.4.1-20211025.jar

    +

    tomcat-jsp-api-9.0.39.jar

    +

    hadoop-yarn-server-common-3.1.1-ei-302002.jar

    +

    kubernetes-model-flowcontrol-5.4.1-20211025.jar

    +

    tomcat-juli-9.0.39.jar

    +

    hadoop-yarn-server-resourcemanager-3.1.1-ei-302002.jar

    +

    kubernetes-model-metrics-5.4.1-20211025.jar

    +

    tomcat-servlet-api-9.0.39.jar

    +

    hadoop-yarn-server-web-proxy-3.1.1-ei-302002.jar

    +

    kubernetes-model-networking-5.4.1-20211025.jar

    +

    tomcat-util-9.0.39.jar

    +

    hamcrest-core-1.3.jar

    +

    kubernetes-model-node-5.4.1-20211025.jar

    +

    tomcat-util-scan-9.0.39.jar

    +

    hive-common-1.2.1-2.0.0.dli-20220528.090500-402.jar

    +

    kubernetes-model-policy-5.4.1-20211025.jar

    +

    univocity-parsers-2.7.3.jar

    +

    hive-exec-1.2.1-2.0.0.dli-20220528.090521-401.jar

    +

    kubernetes-model-rbac-5.4.1-20211025.jar

    +

    zstd-jni-1.4.9-1.jar

    +

    hive-metastore-1.2.1-2.0.0.dli-20220528.090509-402.jar

    +

    kubernetes-model-scheduling-5.4.1-20211025.jar

    +

    validation-api-1.1.0.Final.jar

    +

    hive-shims-0.23-1.2.1-2.0.0.dli-20220528.090445-403.jar

    +

    kubernetes-model-storageclass-5.4.1-20211025.jar

    +

    velocity-1.7.jar

    +

    hive-shims-1.2.1-2.0.0.dli-20220528.090455-403.jar

    +

    leveldbjni-all-1.8-20191105.jar

    +

    woodstox-core-5.0.3.jar

    +

    hive-shims-common-1.2.1-2.0.0.dli-20220528.090441-404.jar

    +

    libfb303-0.9.3.jar

    +

    xbean-asm6-shaded-4.8.jar

    +

    hive-shims-scheduler-1.2.1-2.0.0.dli-20220528.090450-403.jar

    +

    libthrift-0.12.0.jar

    +

    xercesImpl-2.12.0.jar

    +

    hk2-api-2.4.0-b34.jar

    +

    log4j-1.2.17-cloudera1.jar

    +

    xml-apis-1.4.01.jar

    +

    hk2-locator-2.4.0-b34.jar

    +

    log4j-rolling-appender-20131024-2017.jar

    +

    xz-1.0.jar

    +

    hk2-utils-2.4.0-b34.jar

    +

    logging-interceptor-3.14.9.jar

    +

    zjsonpatch-0.3.0.jar

    +

    hppc-0.7.2.jar

    +

    lucene-analyzers-common-7.7.2.jar

    +

    zookeeper-3.5.6-ei-302002.jar

    +

    htrace-core4-4.2.0-incubating-1.0.0.jar

    +

    lucene-core-7.7.2.jar

    +

    -

    +
    +
    +
    +

    Spark 2.3.2 Dependencies

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 3 Spark 2.3.2 dependencies

    Dependency

    +

    accessors-smart-1.2.jar

    +

    HikariCP-java7-2.4.12.jar

    +

    logging-interceptor-3.14.4.jar

    +

    activation-1.1.1.jar

    +

    hive-common-1.2.1-2.1.0.dli-20201111.064115-91.jar

    +

    luxor-encrypt-2.1.0-20201106.065437-53.jar

    +

    aircompressor-0.8.jar

    +

    hive-exec-1.2.1-2.1.0.dli-20201111.064444-91.jar

    +

    luxor-fs3-2.1.0-20201106.065612-53.jar

    +

    alluxio-2.3.1-luxor-SNAPSHOT-client.jar

    +

    hive-metastore-1.2.1-2.1.0.dli-20201111.064230-91.jar

    +

    luxor-obs-fs3-2.1.0-20201106.065616-53.jar

    +

    antlr-2.7.7.jar

    +

    hk2-api-2.4.0-b34.jar

    +

    luxor-rpc_2.11-2.1.0-20201106.065541-53.jar

    +

    antlr4-runtime-4.8-1.jar

    +

    hk2-locator-2.4.0-b34.jar

    +

    luxor-rpc-protobuf2-2.1.0-20201106.065551-53.jar

    +

    antlr-runtime-3.4.jar

    +

    hk2-utils-2.4.0-b34.jar

    +

    lz4-java-1.7.1.jar

    +

    aopalliance-1.0.jar

    +

    hppc-0.7.2.jar

    +

    machinist_2.11-0.6.1.jar

    +

    aopalliance-repackaged-2.4.0-b34.jar

    +

    htrace-core4-4.2.0-incubating-1.0.0.jar

    +

    macro-compat_2.11-1.1.1.jar

    +

    apache-log4j-extras-1.2.17.jar

    +

    httpclient-4.5.4.jar

    +

    metrics-core-3.1.5.jar

    +

    arpack_combined_all-0.1.jar

    +

    httpcore-4.4.7.jar

    +

    metrics-graphite-3.1.5.jar

    +

    arrow-format-0.8.0.jar

    +

    ivy-2.4.0.jar

    +

    metrics-jmx-4.1.12.1.jar

    +

    arrow-memory-0.8.0.jar

    +

    j2objc-annotations-1.3.jar

    +

    metrics-json-3.1.5.jar

    +

    arrow-vector-0.8.0.jar

    +

    jackson-annotations-2.10.0.jar

    +

    metrics-jvm-3.1.5.jar

    +

    asm-5.0.4.jar

    +

    jackson-core-2.10.0.jar

    +

    minlog-1.3.0.jar

    +

    audience-annotations-0.5.0.jar

    +

    jackson-core-asl-1.9.13-atlassian-4.jar

    +

    mssql-jdbc-6.2.1.jre7.jar

    +

    automaton-1.11-8.jar

    +

    jackson-databind-2.10.0.jar

    +

    netty-3.10.6.Final.jar

    +

    avro-1.7.7.jar

    +

    jackson-dataformat-yaml-2.10.0.jar

    +

    netty-all-4.1.51.Final.jar

    +

    avro-ipc-1.7.7.jar

    +

    jackson-datatype-jsr310-2.10.3.jar

    +

    nimbus-jose-jwt-8.19.jar

    +

    avro-ipc-1.7.7-tests.jar

    +

    jackson-jaxrs-base-2.10.3.jar

    +

    objenesis-2.1.jar

    +

    avro-mapred-1.7.7-hadoop2.jar

    +

    jackson-jaxrs-json-provider-2.10.3.jar

    +

    okhttp-3.14.4.jar

    +

    java-sdk-bundle-1.11.271.jar

    +

    jackson-mapper-asl-1.9.13-atlassian-4.jar

    +

    okio-1.17.2.jar

    +

    base64-2.3.8.jar

    +

    jackson-module-jaxb-annotations-2.10.3.jar

    +

    opencsv-2.3.jar

    +

    bcpkix-jdk15on-1.66.jar

    +

    jackson-module-paranamer-2.10.0.jar

    +

    opencsv-4.6.jar

    +

    bcprov-jdk15on-1.66.jar

    +

    jackson-module-scala_2.11-2.10.0.jar

    +

    opencv-4.3.0-2.jar

    +

    bonecp-0.8.0.RELEASE.jar

    +

    jakarta.activation-api-1.2.1.jar

    +

    orc-core-1.4.4-nohive.jar

    +

    breeze_2.11-0.13.2.jar

    +

    jakarta.xml.bind-api-2.3.2.jar

    +

    orc-mapreduce-1.4.4-nohive.jar

    +

    breeze-macros_2.11-0.13.2.jar

    +

    janino-3.0.8.jar

    +

    oro-2.0.8.jar

    +

    calcite-avatica-1.2.0-incubating.jar

    +

    javacpp-1.5.4.jar

    +

    osgi-resource-locator-1.0.1.jar

    +

    calcite-core-1.2.0-incubating.jar

    +

    javacpp-1.5.4-linux-x86_64.jar

    +

    paranamer-2.8.jar

    +

    calcite-linq4j-1.2.0-incubating.jar

    +

    javacv-1.5.4.jar

    +

    parquet-column-1.8.3.jar

    +

    checker-qual-2.11.1.jar

    +

    JavaEWAH-1.1.7.jar

    +

    parquet-common-1.8.3.jar

    +

    chill_2.11-0.8.4.jar

    +

    javassist-3.18.1-GA.jar

    +

    parquet-encoding-1.8.3.jar

    +

    chill-java-0.8.4.jar

    +

    javax.annotation-api-1.2.jar

    +

    parquet-format-2.3.1.jar

    +

    commons-beanutils-1.9.4.jar

    +

    javax.inject-1.jar

    +

    parquet-hadoop-1.8.3.jar

    +

    commons-cli-1.2.jar

    +

    javax.inject-2.4.0-b34.jar

    +

    parquet-hadoop-bundle-1.6.0.jar

    +

    commons-codec-2.0-20130428.202122-59.jar

    +

    javax.servlet-api-3.1.0.jar

    +

    parquet-jackson-1.8.3.jar

    +

    commons-collections-3.2.2.jar

    +

    javax.ws.rs-api-2.0.1.jar

    +

    parquet-format-2.3.1.jar

    +

    commons-collections4-4.2.jar

    +

    java-xmlbuilder-1.1.jar

    +

    parquet-hadoop-1.8.3.jar

    +

    commons-compiler-3.0.8.jar

    +

    javolution-5.3.1.jar

    +

    parquet-hadoop-bundle-1.6.0.jar

    +

    commons-compress-1.4.1.jar

    +

    jaxb-api-2.2.11.jar

    +

    parquet-jackson-1.8.3.jar

    +

    commons-configuration2-2.1.1.jar

    +

    jcip-annotations-1.0-1.jar

    +

    postgresql-42.2.14.jar

    +

    commons-crypto-1.0.0-20191105.jar

    +

    jcl-over-slf4j-1.7.26.jar

    +

    protobuf-java-2.5.0.jar

    +

    commons-daemon-1.0.13.jar

    +

    jdo-api-3.0.1.jar

    +

    py4j-0.10.7.jar

    +

    commons-dbcp-1.4.jar

    +

    jersey-client-2.23.1.jar

    +

    pyrolite-4.13.jar

    +

    commons-dbcp2-2.7.0.jar

    +

    jersey-common-2.23.1.jar

    +

    re2j-1.1.jar

    +

    commons-httpclient-3.1.jar

    +

    jersey-container-servlet-2.23.1.jar

    +

    RoaringBitmap-0.5.11.jar

    +

    commons-io-2.5.jar

    +

    jersey-container-servlet-core-2.23.1.jar

    +

    scala-compiler-2.11.12.jar

    +

    commons-lang-2.6.jar

    +

    jersey-guava-2.23.1.jar

    +

    scala-library-2.11.12.jar

    +

    commons-lang3-3.5.jar

    +

    jersey-media-jaxb-2.23.1.jar

    +

    scalap-2.11.0.jar

    +

    commons-logging-1.2.jar

    +

    jersey-server-2.23.1.jar

    +

    scala-parser-combinators_2.11-1.1.0.jar

    +

    commons-math3-3.4.1.jar

    +

    jets3t-0.9.4.jar

    +

    scala-reflect-2.11.12.jar

    +

    commons-net-2.2.jar

    +

    jetty-http-9.4.31.v20200723.jar

    +

    scala-xml_2.11-1.0.5.jar

    +

    commons-pool-1.5.4.jar

    +

    jetty-io-9.4.31.v20200723.jar

    +

    secComponentApi-1.0.5c.jar

    +

    commons-pool2-2.8.0.jar

    +

    jetty-security-9.4.31.v20200723.jar

    +

    shapeless_2.11-2.3.2.jar

    +

    commons-text-1.3.jar

    +

    jetty-server-9.4.31.v20200723.jar

    +

    slf4j-api-1.7.30.jar

    +

    compress-lzf-1.0.3.jar

    +

    jetty-servlet-9.4.31.v20200723.jar

    +

    slf4j-log4j12-1.7.30.jar

    +

    core-1.1.2.jar

    +

    jetty-util-9.4.31.v20200723.jar

    +

    snakeyaml-1.24.jar

    +

    curator-client-4.2.0.jar

    +

    jetty-util-ajax-9.4.31.v20200723.jar

    +

    snappy-java-1.1.7.5.jar

    +

    curator-framework-4.2.0.jar

    +

    jetty-webapp-9.4.31.v20200723.jar

    +

    spark-catalyst_2.11-2.3.2.0101-2.1.0.dli-20201111.073826-143.jar

    +

    curator-recipes-2.7.1.jar

    +

    jetty-xml-9.4.31.v20200723.jar

    +

    spark-core_2.11-2.3.2.0101-.0.dli-20201111.073836-134.jar

    +

    datanucleus-api-jdo-3.2.6.jar

    +

    joda-time-2.9.3.jar

    +

    spark-graphx_2.11-2.3.2.0101-2.1.0.dli-20201111.073847-129.jar

    +

    datanucleus-core-3.2.10.jar

    +

    jodd-core-4.2.0.jar

    +

    spark-hive_2.11-2.3.2.0101-.0.dli-20201111.073854-132.jar

    +

    datanucleus-rdbms-3.2.9.jar

    +

    json-20200518.jar

    +

    spark-kubernetes_2.11-2.3.2.0101-2.1.0.dli-20201111.073916-85.jar

    +

    derby-10.12.1.1.jar

    +

    json4s-ast_2.11-3.2.11.jar

    +

    spark-kvstore_2.11-2.3.2.0101-2.1.0.dli-20201111.073933-127.jar

    +

    dnsjava-2.1.7.jar

    +

    json4s-core_2.11-3.2.11.jar

    +

    spark-launcher_2.11-2.3.2.0101-2.1.0.dli-20201111.073940-127.jar

    +

    ehcache-3.3.1.jar

    +

    json4s-jackson_2.11-3.2.11.jar

    +

    spark-mllib_2.11-2.3.2.0101-2.1.0.dli-20201111.073946-127.jar

    +

    eigenbase-properties-1.1.5.jar

    +

    json-sanitizer-1.2.1.jar

    +

    spark-mllib-local_2.11-2.3.2.0101-2.1.0.dli-20201111.073953-127.jar

    +

    error_prone_annotations-2.3.4.jar

    +

    json-smart-2.3.jar

    +

    spark-network-common_2.11-2.3.2.0101-2.1.0.dli-20201111.073959-127.jar

    +

    failureaccess-1.0.1.jar

    +

    jsp-api-2.1.jar

    +

    spark-network-shuffle_2.11-2.3.2.0101-2.1.0.dli-20201111.074007-127.jar

    +

    fastutil-8.2.3.jar

    +

    jsr305-3.0.2.jar

    +

    spark-om_2.11-2.3.2.0101-.0.dli-20201111.074019-125.jar

    +

    ffmpeg-4.3.1-1.5.4.jar

    +

    jta-1.1.jar

    +

    spark-repl_2.11-2.3.2.0101-2.1.0.dli-20201111.074028-125.jar

    +

    ffmpeg-4.3.1-1.5.4-linux-x86_64.jar

    +

    jtransforms-2.4.0.jar

    +

    spark-sketch_2.11-2.3.2.0101-2.1.0.dli-20201111.074035-125.jar

    +

    flatbuffers-1.2.0-3f79e055.jar

    +

    jul-to-slf4j-1.7.26.jar

    +

    spark-sql_2.11-2.3.2.0101-2.1.0.dli-20201111.074041-126.jar

    +

    generex-1.0.2.jar

    +

    junit-4.11.jar

    +

    spark-streaming_2.11-2.3.2.0101-2.1.0.dli-20201111.074100-123.jar

    +

    geronimo-jcache_1.0_spec-1.0-alpha-1.jar

    +

    kerb-admin-1.0.1.jar

    +

    spark-tags_2.11-2.3.2.0101-2.1.0.dli-20201111.074136-123.jar

    +

    gson-2.2.4.jar

    +

    kerb-client-1.0.1.jar

    +

    spark-tags_2.11-2.3.2.0101-2.1.0.dli-20201111.074141-124-tests.jar

    +

    guava-29.0-jre.jar

    +

    kerb-common-1.0.1.jar

    +

    spark-unsafe_2.11-2.3.2.0101-2.1.0.dli-20201111.074144-123.jar

    +

    guice-4.0.jar

    +

    kerb-core-1.0.1.jar

    +

    spark-uquery_2.11-2.3.2.0101-2.1.0.dli-20201111.074906-210.jar

    +

    guice-servlet-4.0.jar

    +

    kerb-crypto-1.0.1.jar

    +

    spark-yarn_2.11-2.3.2.0101-2.1.0.dli-20201111.074151-123.jar

    +

    hadoop-annotations-3.1.1-ei-302002.jar

    +

    kerb-identity-1.0.1.jar

    +

    spire_2.11-0.13.0.jar

    +

    hadoop-auth-3.1.1-ei-302002.jar

    +

    kerb-server-1.0.1.jar

    +

    spire-macros_2.11-0.13.0.jar

    +

    hadoop-3.1.1-ei-302002.jar

    +

    kerb-simplekdc-1.0.1.jar

    +

    ST4-4.3.1.jar

    +

    hadoop-client-3.1.1-ei-302002.jar

    +

    kerb-util-1.0.1.jar

    +

    stax2-api-3.1.4.jar

    +

    hadoop-common-3.1.1-ei-302002.jar

    +

    kerby-asn1-1.0.1.jar

    +

    stax-api-1.0-2.jar

    +

    hadoop-hdfs-3.1.1-ei-302002.jar

    +

    kerby-config-1.0.1.jar

    +

    stream-2.7.0.jar

    +

    hadoop-hdfs-client-3.1.1-ei-302002.jar

    +

    kerby-pkix-1.0.1.jar

    +

    stringtemplate-3.2.1.jar

    +

    hadoop-3.1.1-41.jar

    +

    kerby-util-1.0.1.jar

    +

    token-provider-1.0.1.jar

    +

    hadoop-mapreduce-client-common-3.1.1-ei-302002.jar

    +

    kerby-xdr-1.0.1.jar

    +

    univocity-parsers-2.5.9.jar

    +

    hadoop-mapreduce-client-core-3.1.1-ei-302002.jar

    +

    kryo-shaded-3.0.3.jar

    +

    validation-api-1.1.0.Final.jar

    +

    hadoop-mapreduce-client-jobclient-3.1.1-ei-302002.jar

    +

    kubernetes-client-4.9.2-20200804.jar

    +

    woodstox-core-5.0.3.jar

    +

    hadoop-minikdc-3.1.1-ei-302002.jar

    +

    kubernetes-model-4.9.2-20200804.jar

    +

    xbean-asm5-shaded-4.4.jar

    +

    hadoop-yarn-api-3.1.1-ei-302002.jar

    +

    kubernetes-model-common-4.9.2-20200804.jar

    +

    xercesImpl-2.12.0.jar

    +

    hadoop-yarn-client-3.1.1-ei-302002.jar

    +

    leveldbjni-all-1.8-20191105.jar

    +

    xml-apis-1.4.01.jar

    +

    hadoop-yarn-common-3.1.1-ei-302002.jar

    +

    libfb303-0.9.3.jar

    +

    xz-1.0.jar

    +

    hadoop-yarn-registry-3.1.1-ei-302002.jar

    +

    libthrift-0.12.0.jar

    +

    zjsonpatch-0.3.0.jar

    +

    hadoop-yarn-server-common-3.1.1-ei-302002.jar

    +

    listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar

    +

    zookeeper-3.5.6-ei-302002.jar

    +

    hadoop-yarn-server-web-proxy-3.1.1-ei-302002.jar

    +

    log4j-1.2.17-cloudera1.jar

    +

    zookeeper-jute-3.5.6-ei-302002.jar

    +

    hamcrest-core-1.3.jar

    +

    log4j-rolling-appender-20131024-2017.jar

    +

    zstd-jni-1.4.4-11.jar

    +
    +
    +
    +

    Flink 1.12 Dependencies

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 4 Flink 1.12 dependencies

    Dependency

    +

    bcpkix-jdk15on-1.60.jar

    +

    flink-json-1.12.2-ei-313001-dli-2022011002.jar

    +

    libtensorflow-1.12.0.jar

    +

    bcprov-jdk15on-1.60.jar

    +

    flink-kubernetes_2.11-1.12.2-ei-313001-dli-2022011002.jar

    +

    log4j-1.2-api-2.17.1.jar

    +

    clickhouse-jdbc-0.3.1-ei-313001-SNAPSHOT.jar

    +

    flink-metrics-prometheus_2.11-1.12.2-ei-313001-dli-2022011002.jar

    +

    log4j-api-2.17.1.jar

    +

    commons-codec-1.9.jar

    +

    flink-obs-hadoop-fs-2.0.0-20220226.034421-73.jar

    +

    log4j-core-2.17.1.jar

    +

    commons-configuration-1.7.jar

    +

    flink-s3-fs-hadoop-1.12.2.jar

    +

    log4j-slf4j-impl-2.17.1.jar

    +

    dataflow-fs-obs-2.0.0-20220226.034402-190.jar

    +

    flink-shaded-zookeeper-3.6.3-ei-313001-SNAPSHOT.jar

    +

    luxor-encrypt-2.0.0-20220405.072004-199.jar

    +

    deeplearning4j-core-0.9.1.jar

    +

    flink-sql-avro-1.12.2-ei-313001-dli-2022011002.jar

    +

    luxor-fs3-2.0.0-20220405.072025-195.jar

    +

    deeplearning4j-nlp-0.9.1.jar

    +

    flink-sql-avro-confluent-registry-1.12.2-ei-313001-dli-2022011002.jar

    +

    luxor-obs-fs3-2.0.0-20220405.072030-195.jar

    +

    deeplearning4j-nn-0.9.1.jar

    +

    flink-table_2.11-1.12.2-ei-313001-dli-2022011002.jar

    +

    manager-hadoop-security-crypter-8.1.3-313001-SNAPSHOT.jar

    +

    ejml-cdense-0.33.jar

    +

    flink-table-blink_2.11-1.12.2-ei-313001-dli-2022011002.jar

    +

    manager-wc2frm-8.1.3-313001-SNAPSHOT.jar

    +

    ejml-core-0.33.jar

    +

    guava-18.0.jar

    +

    mrs-obs-provider-3.1.1.49.jar

    +

    ejml-ddense-0.33.jar

    +

    guava-26.0-jre.jar

    +

    nd4j-api-0.9.1.jar

    +

    ejml-dsparse-0.33.jar

    +

    hadoop-hdfs-client-3.1.1-ei-302002.jar

    +

    nd4j-native-0.9.1.jar

    +

    ejml-experimental-0.33.jar

    +

    hadoop-3.1.1-46.jar

    +

    nd4j-native-api-0.9.1.jar

    +

    ejml-fdense-0.33.jar

    +

    hadoop-plugins-8.1.3-313001-SNAPSHOT.jar

    +

    nd4j-native-platform-0.9.1.jar

    +

    ejml-simple-0.33.jar

    +

    httpasyncclient-4.1.2.jar

    +

    okhttp-3.14.8.jar

    +

    ejml-zdense-0.33.jar

    +

    httpclient-4.5.3.jar

    +

    okio-1.14.0.jar

    +

    elsa-3.0.0-M7.jar

    +

    httpcore-4.4.4.jar

    +

    ranger-obs-client-0.1.1.jar

    +

    flink-changelog-json-1.12.2-ei-313001-dli-2022011002.jar

    +

    httpcore-nio-4.4.4.jar

    +

    secComponentApi-1.0.5.jar

    +

    flink-csv-1.12.2-ei-313001-dli-2022011002.jar

    +

    java-xmlbuilder-1.1.jar

    +

    slf4j-api-1.7.26.jar

    +

    flink-dist_2.11-1.12.2-ei-313001-dli-2022011002.jar

    +

    jna-4.1.0.jar

    +

    tensorflow-1.12.0.jar

    +
    +

    Flink 1.10 Dependencies

    Only queues created after December 2020 can use the Flink 1.10 dependencies.

    -
    • bcpkix-jdk15on-1.60.jar
    • bcprov-jdk15on-1.60.jar
    • commons-codec-1.9.jar
    • commons-configuration-1.7.jar
    • deeplearning4j-core-0.9.1.jar
    • deeplearning4j-nlp-0.9.1.jar
    • deeplearning4j-nn-0.9.1.jar
    • ejml-cdense-0.33.jar
    • ejml-core-0.33.jar
    • ejml-ddense-0.33.jar
    • ejml-dsparse-0.33.jar
    • ejml-experimental-0.33.jar
    • ejml-fdense-0.33.jar
    • ejml-simple-0.33.jar
    • ejml-zdense-0.33.jar
    • elsa-3.0.0-M7.jar
    • esdk-obs-java-3.20.6.1.jar
    • flink-cep_2.11-1.10.0.jar
    • flink-cep-scala_2.11-1.10.0.jar
    • flink-dist_2.11-1.10.0.jar
    • flink-python_2.11-1.10.0.jar
    • flink-queryable-state-runtime_2.11-1.10.0.jar
    • flink-sql-client_2.11-1.10.0.jar
    • flink-state-processor-api_2.11-1.10.0.jar
    • flink-table_2.11-1.10.0.jar
    • flink-table-blink_2.11-1.10.0.jar
    • guava-26.0-jre.jar
    • hadoop-3.1.1-41.jar
    • httpasyncclient-4.1.2.jar
    • httpclient-4.5.3.jar
    • httpcore-4.4.4.jar
    • httpcore-nio-4.4.4.jar
    • java-xmlbuilder-1.1.jar
    • jna-4.1.0.jar
    • libtensorflow-1.12.0.jar
    • log4j-over-slf4j-1.7.26.jar
    • logback-classic-1.2.3.jar
    • logback-core-1.2.3.jar
    • nd4j-api-0.9.1.jar
    • nd4j-native-0.9.1.jar
    • nd4j-native-api-0.9.1.jar
    • nd4j-native-platform-0.9.1.jar
    • okhttp-3.14.8.jar
    • okio-1.14.0.jar
    • secComponentApi-1.0.5.jar
    • slf4j-api-1.7.26.jar
    • tensorflow-1.12.0.jar
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 5 Flink 1.10 dependencies

    Dependency

    +

    bcpkix-jdk15on-1.60.jar

    +

    esdk-obs-java-3.20.6.1.jar

    +

    java-xmlbuilder-1.1.jar

    +

    bcprov-jdk15on-1.60.jar

    +

    flink-cep_2.11-1.10.0.jar

    +

    jna-4.1.0.jar

    +

    commons-codec-1.9.jar

    +

    flink-cep-scala_2.11-1.10.0.jar

    +

    libtensorflow-1.12.0.jar

    +

    commons-configuration-1.7.jar

    +

    flink-dist_2.11-1.10.0.jar

    +

    log4j-over-slf4j-1.7.26.jar

    +

    deeplearning4j-core-0.9.1.jar

    +

    flink-python_2.11-1.10.0.jar

    +

    logback-classic-1.2.3.jar

    +

    deeplearning4j-nlp-0.9.1.jar

    +

    flink-queryable-state-runtime_2.11-1.10.0.jar

    +

    logback-core-1.2.3.jar

    +

    deeplearning4j-nn-0.9.1.jar

    +

    flink-sql-client_2.11-1.10.0.jar

    +

    nd4j-api-0.9.1.jar

    +

    ejml-cdense-0.33.jar

    +

    flink-state-processor-api_2.11-1.10.0.jar

    +

    nd4j-native-0.9.1.jar

    +

    ejml-core-0.33.jar

    +

    flink-table_2.11-1.10.0.jar

    +

    nd4j-native-api-0.9.1.jar

    +

    ejml-ddense-0.33.jar

    +

    flink-table-blink_2.11-1.10.0.jar

    +

    nd4j-native-platform-0.9.1.jar

    +

    ejml-dsparse-0.33.jar

    +

    guava-26.0-jre.jar

    +

    okhttp-3.14.8.jar

    +

    ejml-experimental-0.33.jar

    +

    hadoop-3.1.1-41.jar

    +

    okio-1.14.0.jar

    +

    ejml-fdense-0.33.jar

    +

    httpasyncclient-4.1.2.jar

    +

    secComponentApi-1.0.5.jar

    +

    ejml-simple-0.33.jar

    +

    httpclient-4.5.3.jar

    +

    slf4j-api-1.7.26.jar

    +

    ejml-zdense-0.33.jar

    +

    httpcore-4.4.4.jar

    +

    tensorflow-1.12.0.jar

    +

    elsa-3.0.0-M7.jar

    +

    httpcore-nio-4.4.4.jar

    +

    -

    +
    +
    +
    +

    Flink 1.7.2 Dependencies

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Table 6 Flink 1.7.2 dependencies

    Dependency

    +

    bcpkix-jdk15on-1.60.jar

    +

    esdk-obs-java-3.1.3.jar

    +

    httpcore-4.4.4.jar

    +

    bcprov-jdk15on-1.60.jar

    +

    flink-cep_2.11-1.7.0.jar

    +

    httpcore-nio-4.4.4.jar

    +

    commons-codec-1.9.jar

    +

    flink-cep-scala_2.11-1.7.0.jar

    +

    java-xmlbuilder-1.1.jar

    +

    commons-configuration-1.7.jar

    +

    flink-dist_2.11-1.7.0.jar

    +

    jna-4.1.0.jar

    +

    deeplearning4j-core-0.9.1.jar

    +

    flink-gelly_2.11-1.7.0.jar

    +

    libtensorflow-1.12.0.jar

    +

    deeplearning4j-nlp-0.9.1.jar

    +

    flink-gelly-scala_2.11-1.7.0.jar

    +

    log4j-over-slf4j-1.7.21.jar

    +

    deeplearning4j-nn-0.9.1.jar

    +

    flink-ml_2.11-1.7.0.jar

    +

    logback-classic-1.2.3.jar

    +

    ejml-cdense-0.33.jar

    +

    flink-python_2.11-1.7.0.jar

    +

    logback-core-1.2.3.jar

    +

    ejml-core-0.33.jar

    +

    flink-queryable-state-runtime_2.11-1.7.0.jar

    +

    nd4j-api-0.9.1.jar

    +

    ejml-ddense-0.33.jar

    +

    flink-shaded-curator-1.7.0.jar

    +

    nd4j-native-0.9.1.jar

    +

    ejml-dsparse-0.33.jar

    +

    flink-shaded-hadoop2-uber-1.7.0.jar

    +

    nd4j-native-api-0.9.1.jar

    +

    ejml-experimental-0.33.jar

    +

    flink-table_2.11-1.7.0.jar

    +

    nd4j-native-platform-0.9.1.jar

    +

    ejml-fdense-0.33.jar

    +

    guava-26.0-jre.jar

    +

    okhttp-3.14.8.jar

    +

    ejml-simple-0.33.jar

    +

    hadoop-3.1.1-41-20201014.085840-4.jar

    +

    okio-1.14.0.jar

    +

    ejml-zdense-0.33.jar

    +

    httpasyncclient-4.1.2.jar

    +

    slf4j-api-1.7.21.jar

    +

    elsa-3.0.0-M7.jar

    +

    httpclient-4.5.12.jar

    +

    tensorflow-1.12.0.jar

    +
    -

    Flink 1.12 Dependencies

    • bcpkix-jdk15on-1.60.jar
    • bcprov-jdk15on-1.60.jar
    • clickhouse-jdbc-0.3.1-ei-313001-SNAPSHOT.jar
    • commons-codec-1.9.jar
    • commons-configuration-1.7.jar
    • dataflow-fs-obs-2.0.0-20220226.034402-190.jar
    • deeplearning4j-core-0.9.1.jar
    • deeplearning4j-nlp-0.9.1.jar
    • deeplearning4j-nn-0.9.1.jar
    • ejml-cdense-0.33.jar
    • ejml-core-0.33.jar
    • ejml-ddense-0.33.jar
    • ejml-dsparse-0.33.jar
    • ejml-experimental-0.33.jar
    • ejml-fdense-0.33.jar
    • ejml-simple-0.33.jar
    • ejml-zdense-0.33.jar
    • elsa-3.0.0-M7.jar
    • flink-changelog-json-1.12.2-ei-313001-dli-2022011002.jar
    • flink-csv-1.12.2-ei-313001-dli-2022011002.jar
    • flink-dist_2.11-1.12.2-ei-313001-dli-2022011002.jar
    • flink-json-1.12.2-ei-313001-dli-2022011002.jar
    • flink-kubernetes_2.11-1.12.2-ei-313001-dli-2022011002.jar
    • flink-metrics-prometheus_2.11-1.12.2-ei-313001-dli-2022011002.jar
    • flink-obs-hadoop-fs-2.0.0-20220226.034421-73.jar
    • flink-s3-fs-hadoop-1.12.2.jar
    • flink-shaded-zookeeper-3.6.3-ei-313001-SNAPSHOT.jar
    • flink-sql-avro-1.12.2-ei-313001-dli-2022011002.jar
    • flink-sql-avro-confluent-registry-1.12.2-ei-313001-dli-2022011002.jar
    • flink-table_2.11-1.12.2-ei-313001-dli-2022011002.jar
    • flink-table-blink_2.11-1.12.2-ei-313001-dli-2022011002.jar
    • guava-18.0.jar
    • guava-26.0-jre.jar
    • hadoop-hdfs-client-3.1.1-ei-302002.jar
    • hadoop-3.1.1-46.jar
    • hadoop-plugins-8.1.3-313001-SNAPSHOT.jar
    • httpasyncclient-4.1.2.jar
    • httpclient-4.5.3.jar
    • httpcore-4.4.4.jar
    • httpcore-nio-4.4.4.jar
    • java-xmlbuilder-1.1.jar
    • jna-4.1.0.jar
    • libtensorflow-1.12.0.jar
    • log4j-1.2-api-2.17.1.jar
    • log4j-api-2.17.1.jar
    • log4j-core-2.17.1.jar
    • log4j-slf4j-impl-2.17.1.jar
    • luxor-encrypt-2.0.0-20220405.072004-199.jar
    • luxor-fs3-2.0.0-20220405.072025-195.jar
    • luxor-obs-fs3-2.0.0-20220405.072030-195.jar
    • manager-hadoop-security-crypter-8.1.3-313001-SNAPSHOT.jar
    • manager-wc2frm-8.1.3-313001-SNAPSHOT.jar
    • mrs-obs-provider-3.1.1.49.jar
    • nd4j-api-0.9.1.jar
    • nd4j-native-0.9.1.jar
    • nd4j-native-api-0.9.1.jar
    • nd4j-native-platform-0.9.1.jar
    • okhttp-3.14.8.jar
    • okio-1.14.0.jar
    • ranger-obs-client-0.1.1.jar
    • secComponentApi-1.0.5.jar
    • slf4j-api-1.7.26.jar
    • tensorflow-1.12.0.jar
    diff --git a/docs/dli/umn/dli_01_0402.html b/docs/dli/umn/dli_01_0402.html index 57e835a0b..5368bf346 100644 --- a/docs/dli/umn/dli_01_0402.html +++ b/docs/dli/umn/dli_01_0402.html @@ -7,7 +7,7 @@

    Constraints

    • A queue named default is preset in DLI for you to experience. Resources are allocated on demand.
    • Queue types:
      • For SQL: Spark SQL jobs can be submitted to SQL queues.
      • For general purpose: The queue is used to run Spark programs, Flink SQL jobs, and Flink Jar jobs.

      The queue type cannot be changed. If you want to use another queue type, purchase a new queue.

      -
    • The region of a queue cannot be changed.
    • A newly created queue can be scaled in or out only after a job is executed on the queue.
    • DLI queues cannot access the Internet.

      +
    • The region of a queue cannot be changed.
    • Queues with 16 CUs do not support scale-out or scale-in.
    • Queues with 64 CUs do not support scale-in.
    • A newly created queue can be scaled in or out only after a job is executed on the queue.
    • DLI queues cannot access the Internet.

    diff --git a/docs/dli/umn/dli_01_0403.html b/docs/dli/umn/dli_01_0403.html index ebde0b65b..bf55a6e39 100644 --- a/docs/dli/umn/dli_01_0403.html +++ b/docs/dli/umn/dli_01_0403.html @@ -2,6 +2,7 @@

    Overview

    On the Job Management page of Flink jobs, you can submit a Flink job. Currently, the following job types are supported:

    +
    • Flink SQL uses SQL statements to define jobs and can be submitted to any general purpose queue.
    • Flink Jar customizes a JAR package job based on Flink APIs. It runs on dedicated queues.

    Flink job management provides the following functions:

    @@ -17,62 +18,62 @@

    Flink Jobs Page

    On the Overview page, click Flink Jobs to go to the Flink job management page. Alternatively, you can choose Job Management > Flink Jobs from the navigation pane on the left. The page displays all Flink jobs. If there are a large number of jobs, they will be displayed on multiple pages. DLI allows you to view jobs in all statuses.

    -
    Table 1 Job management parameters

    Parameter

    +
    - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/dli/umn/dli_01_0421.html b/docs/dli/umn/dli_01_0421.html index e7b695cf3..9b18a6146 100644 --- a/docs/dli/umn/dli_01_0421.html +++ b/docs/dli/umn/dli_01_0421.html @@ -1,7 +1,8 @@ -

    Creating a Message Notification Topic

    -

    Scenario

    Once you have created a message notification topic, you can Add subscription of the topic on the Topic Management page of the Simple Message Notification service. You can select different ways (such as text messages or emails) to subscribe. After the subscription succeeds, any job failure will automatically be sent to your subscription endpoints.

    +

    Creating an SMN Topic

    +

    Scenario

    Once you have created an SMN topic, you can easily subscribe to it by going to the Topic Management > Topics page of the SMN console. You can choose to receive notifications via SMS or email. After the subscription is successful, if a job fails, the system automatically sends a message to the subscription endpoint you specified.

    +
    • If a job fails within 1 minute of submission, a message notification is not triggered.
    • If a job fails after 1 minute of submission, the system automatically sends a message to the subscriber terminal you specified.

    Procedure

    1. On the Resources > Queue Management page, click Create SMN Topic on the upper left side. The Create SMN Topic dialog box is displayed.
    2. Select a queue and click OK.
      • You can select a single queue or all queues.
      • If you create a topic for a queue and another topic for all queues, the SMN of all queues does not include the message of the single queue.
      • After a message notification topic is created, you will receive a message notification only when a Spark job created on the subscription queue fails.
      diff --git a/docs/dli/umn/dli_01_0422.html b/docs/dli/umn/dli_01_0422.html index a96139398..ddfaa2934 100644 --- a/docs/dli/umn/dli_01_0422.html +++ b/docs/dli/umn/dli_01_0422.html @@ -4,7 +4,7 @@
        -
      • Introduction
        +
      • Overview
      • Creating a CSS Datasource Authentication
        • diff --git a/docs/dli/umn/dli_01_0440.html b/docs/dli/umn/dli_01_0440.html index 6126c8142..c3bce4b86 100644 --- a/docs/dli/umn/dli_01_0440.html +++ b/docs/dli/umn/dli_01_0440.html @@ -14,43 +14,54 @@
        Type: There are roles and policies.
        • Roles: A type of coarse-grained authorization mechanism that defines permissions related to user responsibilities. Only a limited number of service-level roles are available. When using roles to grant permissions, you also need to assign other roles on which the permissions depend. However, roles are not an ideal choice for fine-grained authorization and secure access control.
        • Policies: A type of fine-grained authorization mechanism that defines permissions required to perform operations on specific cloud resources under certain conditions. This mechanism allows for more flexible policy-based authorization, meeting requirements for secure access control. For example, you can grant DLI users only the permissions for managing a certain type of ECSs.
        -
    Table 1 Job management parameters

    Parameter

    Description

    +

    Description

    ID

    +

    ID

    ID of a submitted Flink job, which is generated by the system by default.

    +

    ID of a submitted Flink job, which is generated by the system by default.

    Name

    +

    Name

    Name of the submitted Flink job.

    +

    Name of the submitted Flink job.

    Type

    +

    Type

    Type of the submitted Flink job. Including:

    +

    Type of the submitted Flink job. Including:

    • Flink SQL: Flink SQL jobs
    • Flink Jar: Flink Jar jobs

    Status

    +

    Status

    Job statuses, including:

    +

    Job statuses, including:

    • Draft
    • Submitting
    • Submission failed
    • Running: After the job is submitted, a normal result is returned.
    • Running exception: The job stops running due to an exception.
    • Downloading
    • Idle
    • Stopping
    • Stopped
    • Stopping failed
    • Creating the savepoint
    • Completed

    Description

    +

    Description

    Description of the submitted Flink job.

    +

    Description of the submitted Flink job.

    Username

    +

    Username

    Name of the user who submits a job.

    +

    Name of the user who submits a job.

    Created

    +

    Created

    Time when a job is created.

    +

    Time when a job is created.

    Started

    +

    Started

    Time when a Flink job starts to run.

    +

    Time when a Flink job starts to run.

    Duration

    +

    Duration

    Time consumed by job running.

    +

    Time consumed by job running.

    Operation

    +

    Operation

    • Edit: Edit a created job. For details, see Editing a Job.
    • Start: Start and run a job. For details, see Starting a Job.
    • More
      • FlinkUI: After you click this button, the Flink job execution page is displayed.
        NOTE:

        When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click FlinkUI before the cluster is created, an empty projectID will be cached. The FlinkUI page cannot be displayed.

        +
    • Edit: Edit a created job. For details, see Editing a Job.
    • Start: Start and run a job. For details, see Starting a Job.
    • More
      • FlinkUI: After you click this button, the Flink job execution page is displayed.
        NOTE:

        When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click FlinkUI before the cluster is created, an empty projectID will be cached. The FlinkUI page cannot be displayed.

        You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check FlinkUI.

      • Stop: Stop a Flink job. If this function is unavailable, jobs in the current status cannot be stopped.
      • Delete: Delete a job.
        NOTE:

        A deleted job cannot be restored.

        diff --git a/docs/dli/umn/dli_01_0410.html b/docs/dli/umn/dli_01_0410.html index 297c306de..8b788b4ae 100644 --- a/docs/dli/umn/dli_01_0410.html +++ b/docs/dli/umn/dli_01_0410.html @@ -53,7 +53,7 @@

    √

    DDS Mongo

    +

    DDS

    √

    Table 1 DLI system permissions

    Role/Policy Name

    +
    - - + - - - + - - - + - - - + - - - + @@ -60,9 +71,9 @@

    DLI Permission Types

    Table 2 lists the DLI service permissions. For details about the resources that can be controlled by DLI, see Table 4.

    -
    Table 1 DLI system permissions

    Role/Policy Name

    Description

    +

    Description

    Category

    +

    Category

    +

    Dependency

    DLI FullAccess

    +

    DLI FullAccess

    Full permissions for DLI.

    +

    Full permissions for DLI.

    System-defined policy

    +

    System-defined policy

    +

    This role depends on other roles in the same project.

    +
    • Creating a datasource connection: VPC ReadOnlyAccess
    • Creating a tag: TMS FullAccess and EPS EPS FullAccess
    • Using OBS for storage: OBS OperateAccess
    • Creating an agency: Security Administrator

    DLI ReadOnlyAccess

    +

    DLI ReadOnlyAccess

    Read-only permissions for DLI.

    -

    With read-only permissions, you can use DLI resources and perform operations that do not require fine-grained permissions. For example, create global variables, create packages and package groups, submit jobs to the default queue, create tables in the default database, create datasource connections, and delete datasource connections.

    +

    Read-only permissions for DLI.

    +

    With read-only permissions, you can use DLI resources and perform operations that do not require fine-grained permissions. For example, create global variables, create packages and package groups, submit jobs to the default queue, create tables in the default database, create datasource connections, and delete datasource connections.

    System-defined policy

    +

    System-defined policy

    +

    None

    Tenant Administrator

    +

    Tenant Administrator

    Tenant administrator

    -
    • Administer permissions for managing and accessing all cloud services. After a database or a queue is created, the user can use the ACL to assign rights to other users.
    • Scope: project-level service
    +

    Tenant administrator

    +
    • Job execution permissions for DLI resources. After a database or a queue is created, the user can use the ACL to assign rights to other users.
    • Scope: project-level service

    System-defined role

    +

    System-defined role

    +

    None

    DLI Service Admin

    +

    DLI Service Administrator

    DLI administrator

    -
    • Administer permissions for managing and accessing the queues and data of DLI. After a database or a queue is created, the user can use the ACL to assign rights to other users.
    • Scope: project-level service
    +

    DLI administrator.

    +
    • Job execution permissions for DLI resources. After a database or a queue is created, the user can use the ACL to assign rights to other users.
    • Scope: project-level service

    System-defined role

    +

    System-defined role

    +

    None
    Table 2 DLI permission types

    Permission Type

    +
    - @@ -72,9 +83,9 @@ - - @@ -86,9 +97,9 @@ - - @@ -103,9 +114,9 @@ - - @@ -114,10 +125,10 @@ - - @@ -129,9 +140,9 @@ - - @@ -145,7 +156,7 @@

    Examples

    An Internet company mainly provides game and music services. DLI is used to analyze user behaviors and assist decision making.

    -

    As shown in Figure 1, the Leader of the Basic Platform Team has applied for a Tenant Administrator account to manage and use cloud services. Since the Big Data Platform Team needs DLI for data analysis, the Leader of the Basic Platform Team adds a subaccount with the permission of DLI Service Admin to manage and use DLI. The Leader of the Basic Platform Team creates a Queue A and assigns it to Data Engineer A to analyze the gaming data. A Queue B is also assigned to Data Engineer B to analyze the music data. Besides granting the queue usage permission, the Leader of the Basic Platform Team grants data (except the database) management and usage permissions to the two engineers.

    +

    As shown in Figure 1, the Leader of the Basic Platform Team has applied for a Tenant Administrator account to manage and use cloud services. The Leader of the Basic Platform Team creates a subaccount with the DLI Service Administrator permission to manage and use DLI, as the Big Data Platform Team requires DLI for data analysis. The Leader of the Basic Platform Team creates a Queue A and assigns it to Data Engineer A to analyze the gaming data. A Queue B is also assigned to Data Engineer B to analyze the music data. Besides granting the queue usage permission, the Leader of the Basic Platform Team grants data (except the database) management and usage permissions to the two engineers.

    Figure 1 Granting permissions

    The Data Engineer A creates a table named gameTable for storing game prop data and a table named userTable for storing game user data. The music service is a new service. To explore potential music users among existing game users, the Data Engineer A assigns the query permission on the userTable to the Data Engineer B. In addition, Data Engineer B creates a table named musicTable for storing music copyrights information.

    Table 3 describes the queue and data permissions of Data Engineer A and Data Engineer B.

    diff --git a/docs/dli/umn/dli_01_0441.html b/docs/dli/umn/dli_01_0441.html index c97682da2..65511366b 100644 --- a/docs/dli/umn/dli_01_0441.html +++ b/docs/dli/umn/dli_01_0441.html @@ -15,7 +15,7 @@
    - diff --git a/docs/dli/umn/dli_01_0445.html b/docs/dli/umn/dli_01_0445.html index f30a94332..59ecdd159 100644 --- a/docs/dli/umn/dli_01_0445.html +++ b/docs/dli/umn/dli_01_0445.html @@ -6,325 +6,377 @@

    Namespace

    SYS.DLI

    Metric

    -
    Table 2 DLI permission types

    Permission Type

    Subtype

    +

    Subtype

    Console Operations

    Queue Permissions

    +

    Queue Permissions

    Queue management permissions

    +

    Queue management permissions

    For details, see Queue Permission Management.

    Queue usage permission

    Data Permissions

    +

    Data Permissions

    Database permissions

    +

    Database permissions

    For details, see Managing Database Permissions and Managing Table Permissions.

    Column permissions

    Job Permissions

    +

    Job Permissions

    Flink job permissions

    +

    Flink job permissions

    For details, see Managing Flink Job Permissions.

    For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

    Package Permissions

    +

    Package Permissions

    Package group permissions

    +

    Package group permissions

    For details, see Managing Permissions on Packages and Package Groups.

    Package permissions

    Datasource Connection Permissions

    +

    Datasource Connection Permissions

    Datasource connection permissions

    +

    Datasource connection permissions

    For details, see Datasource Authentication Permission Management.

    Tenant Administrator

    DLI Service Admin

    +

    DLI Service Administrator
    Table 1 DLI metrics

    Metric ID

    +
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/dli/umn/dli_01_0447.html b/docs/dli/umn/dli_01_0447.html index 45e8f7f7e..b64b33d38 100644 --- a/docs/dli/umn/dli_01_0447.html +++ b/docs/dli/umn/dli_01_0447.html @@ -1,7 +1,7 @@

    Managing Database Permissions

    -

    Scenario

    • You can isolate databases allocated to different users by setting permissions to ensure data query performance.
    • The administrator and database owner have all permissions, which cannot be set or modified by other users.
    +

    Scenario

    • By setting permissions, you can assign varying database permissions to different users.
    • The administrator and database owner have all permissions, which cannot be set or modified by other users.

    Precautions

    • Lower-level objects automatically inherit permissions granted to upper-level objects. The hierarchical relationship is database > table > column.
    • The database owner, table owner, and authorized users can assign permissions on the database and tables.
    • Columns can only inherit the query permission. For details about Inheritable Permissions, see Managing Database Permissions.
    • The permissions can be revoked only at the initial level to which the permissions are granted. You need to grant and revoke permissions at the same level. You need to grant and revoke permissions at the same level. For example, after you are granted the insertion permission on a database, you can obtain the insertion permission on the tables in the database. Your insertion permission can be revoked only at the database level.
    • If you create a database with the same name as a deleted database, the database permissions will not be inherited. In this case, you need to grant the database permissions to users or projects.

      For example, user A is granted with the permission to delete the testdb database. Delete the database and create another one with the same name. You need to grant user A the deletion permission of the testdb database again.

    diff --git a/docs/dli/umn/dli_01_0448.html b/docs/dli/umn/dli_01_0448.html index aef0417a1..3d2b929c9 100644 --- a/docs/dli/umn/dli_01_0448.html +++ b/docs/dli/umn/dli_01_0448.html @@ -1,7 +1,7 @@

    Managing Table Permissions

    -

    Operation Scenario

    • You can isolate databases allocated to different users by setting permissions to ensure data query performance.
    • The administrator and database owner have all permissions, which cannot be set or modified by other users.
    • When setting database permissions for a new user, ensure that the user group to which the user belongs has the Tenant Guest permission.
    +

    Operation Scenario

    • By setting permissions, you can assign varying table permissions to different users.
    • The administrator and table owner have all permissions, which cannot be set or modified by other users.
    • When setting table permissions for a new user, ensure that the user group the user belongs to has the Tenant Guest permission.

    Precautions

    • If you create a table with the same name as a deleted table, the table permissions will not be inherited. In this case, you need to grant the table permissions to users or projects.

      For example, user A is granted with the permission to delete the testTable table. Delete the table and create another one with the same name. You need to grant user A the deletion permission of the testTable table again.

    @@ -55,21 +55,21 @@
    - - - -
    Table 1 DLI metrics

    Metric ID

    Name

    +

    Name

    Description

    Value Range

    +

    Value Range

    Monitored Object

    +

    Monitored Object

    Monitoring Period (Raw Data)

    +

    Monitoring Period (Raw Data)

    queue_cu_num

    +

    queue_cu_num

    CU usage of a queue

    +

    Queue CU Usage

    Displays the number of CUs applied by the user queue

    ≥ 0

    +

    ≥ 0

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_job_launching_num

    +

    queue_job_launching_num

    Number of Jobs Being Submitted

    +

    Number of Jobs Being Submitted

    Displays the number of jobs in the Submitting state in the user queue.

    ≥ 0

    +

    ≥ 0

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_job_running_num

    +

    queue_job_running_num

    Number of Running Jobs

    +

    Number of Running Jobs

    Displays the number of running jobs in the user queue.

    ≥ 0

    +

    ≥ 0

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_job_succeed_num

    +

    queue_job_succeed_num

    Number of Finished Jobs

    +

    Number of Finished Jobs

    Displays the number of completed jobs in the user queue.

    ≥ 0

    +

    ≥ 0

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_job_failed_num

    +

    queue_job_failed_num

    Failed Jobs

    +

    Failed Jobs

    Displays the number of failed jobs in the user queue.

    ≥ 0

    +

    ≥ 0

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_job_cancelled_num

    +

    queue_job_cancelled_num

    Number of Canceled Jobs

    +

    Number of Canceled Jobs

    Displays the number of canceled jobs in the user queue.

    ≥ 0

    +

    ≥ 0

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_cpu_usage

    +

    queue_alloc_cu_num

    Queue CPU Usage

    +

    Allocated CUs (queue)

    +

    Displays the CU allocation for user queues.

    +

    ≥ 0

    +

    Queues

    +

    5 minutes

    +

    queue_min_cu_num

    +

    Minimum CUs for Queue

    +

    Displays the minimum number of CUs for a user queue.

    +

    ≥ 0

    +

    Queues

    +

    5 minutes

    +

    queue_max_cu_num

    +

    Maximum CUs for Queue

    +

    Displays the maximum number of CUs for a user queue.

    +

    ≥ 0

    +

    Queues

    +

    5 minutes

    +

    queue_priority

    +

    Queue Priority

    +

    Displays the priority of a user queue.

    +

    1–100

    +

    Queues

    +

    5 minutes

    +

    queue_cpu_usage

    +

    Queue CPU Usage

    Displays the CPU usage of user queues.

    0–100

    +

    0–100

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_disk_usage

    +

    queue_disk_usage

    Queue Disk Usage

    +

    Queue Disk Usage

    Displays the disk usage of user queues.

    0–100

    +

    0–100

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_disk_used

    +

    queue_disk_used

    Max Disk Usage

    +

    Max Disk Usage

    Displays the maximum disk usage of user queues.

    0~100

    +

    0–100

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_mem_usage

    +

    queue_mem_usage

    Queue Memory Usage

    +

    Queue Memory Usage

    Displays the memory usage of user queues.

    0–100

    +

    0–100

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    queue_mem_used

    +

    queue_mem_used

    Used Memory

    +

    Used Memory

    Displays the memory usage rate of the user queues.

    ≥ 0

    +

    ≥ 0

    Queues

    +

    Queues

    5 minutes

    +

    5 minutes

    flink_read_records_per_second

    +

    flink_read_records_per_second

    Flink Job Data Read Rate

    +

    Flink Job Data Read Rate

    Displays the data input rate of a Flink job for monitoring and debugging.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_write_records_per_second

    +

    flink_write_records_per_second

    Flink Job Data Write Rate

    +

    Flink Job Data Write Rate

    Displays the data output rate of a Flink job for monitoring and debugging.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_read_records_total

    +

    flink_read_records_total

    Flink Job Total Data Read

    +

    Flink Job Total Data Read

    Displays the total number of data inputs of a Flink job for monitoring and debugging.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_write_records_total

    +

    flink_write_records_total

    Flink Job Total Data Write

    +

    Flink Job Total Data Write

    Displays the total number of output data records of a Flink job for monitoring and debugging.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_read_bytes_per_second

    +

    flink_read_bytes_per_second

    Flink Job Byte Read Rate

    +

    Flink Job Byte Read Rate

    Displays the number of input bytes per second of a Flink job.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_write_bytes_per_second

    +

    flink_write_bytes_per_second

    Flink Job Byte Write Rate

    +

    Flink Job Byte Write Rate

    Displays the number of output bytes per second of a Flink job.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_read_bytes_total

    +

    flink_read_bytes_total

    Flink Job Total Read Byte

    +

    Flink Job Total Read Byte

    Displays the total number of input bytes of a Flink job.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_write_bytes_total

    +

    flink_write_bytes_total

    Flink Job Total Write Byte

    +

    Flink Job Total Write Byte

    Displays the total number of output bytes of a Flink job.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_cpu_usage

    +

    flink_cpu_usage

    Flink Job CPU Usage

    +

    Flink Job CPU Usage

    Displays the CPU usage of Flink jobs.

    0–100

    +

    0–100

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_mem_usage

    +

    flink_mem_usage

    Flink Job Memory Usage

    +

    Flink Job Memory Usage

    Displays the memory usage of Flink jobs.

    0–100

    +

    0–100

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_max_op_latency

    +

    flink_max_op_latency

    Flink Job Max Operator Latency

    +

    Flink Job Max Operator Latency

    Displays the maximum operator delay of a Flink job. The unit is ms.

    ≥ 0

    +

    ≥ 0

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    flink_max_op_backpressure_level

    +

    flink_max_op_backpressure_level

    Flink Job Maximum Operator Backpressure

    +

    Flink Job Maximum Operator Backpressure

    Displays the maximum operator backpressure value of a Flink job. A larger value indicates severer backpressure.

    0: OK

    50: low

    100: high

    0–100

    +

    0–100

    Flink jobs

    +

    Flink jobs

    10 seconds

    +

    10 seconds

    Authorization Object

    Select User or Project.

    +

    Select User or Project.

    Username/Project

    • If you select User, enter the IAM username when granting table permissions to the user.
      NOTE:

      The username is an existing IAM user name and has logged in to the DLI management console.

      +
    • If you select User, enter the IAM username when granting table permissions to the user.
      NOTE:

      The username is an existing IAM user name and has logged in to the DLI management console.

      -
    • If you select Project, select the project to be authorized in the current region.
      NOTE:

      If you select Project, you can only view information about the authorized tables and their databases.

      +
    • If you select Project, select the project to be authorized in the current region.
      NOTE:

      If you select Project, you can only view information about the authorized tables and their databases.

    Non-inheritable Permissions

    Select a permission to grant it to the user, or deselect a permission to revoke it.
    • The following permissions are applicable to both user and project authorization:
      • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
      • View Table Information: This permission allows you to view information about the current table.
      • Select Table: This permission allows you to query data of the current table.
      • Drop Table: This permission allows you to delete the current table.
      • Rename Table: Rename the current table.
      • Insert: This permission allows you to insert data into the current table.
      • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
      • Add Column: This permission allows you to add columns to the current table.
      • Grant Permission: This permission allows you to grant table permissions to other users or projects.
      • Revoke Permission: This permission allows you to revoke the table's permissions that other users or projects have but cannot revoke the table owner's permissions.
      • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
      +
    Select a permission to grant it to the user, or deselect a permission to revoke it.
    • The following permissions are applicable to both user and project authorization:
      • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
      • View Table Information: This permission allows you to view information about the current table.
      • Select Table: This permission allows you to query data of the current table.
      • Drop Table: This permission allows you to delete the current table.
      • Rename Table: Rename the current table.
      • Insert: This permission allows you to insert data into the current table.
      • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
      • Add Column: This permission allows you to add columns to the current table.
      • Grant Permission: This permission allows you to grant table permissions to other users or projects.
      • Revoke Permission: This permission allows you to revoke the table's permissions that other users or projects have but cannot revoke the table owner's permissions.
      • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
      The partition table also has the following permissions:
      • Add Partition: This permission allows you to add a partition to a partition table.
      • Delete Partition: This permission allows you to delete existing partitions from a partition table.
      • Configure Path for Partition: This permission allows you to set the path of a partition in a partition table to a specified OBS path.
      • Rename Table Partition: This permission allows you to rename partitions in a partition table.
      • Restore Table Partition: This permission allows you to export partition information from the file system and save the information to metadata.
      • View All Partitions: This permission allows you to view all partitions in a partition table.
    @@ -92,7 +92,7 @@

    Authorization Object

    Select User or Project.

    +

    Select User or Project.

    Username/Project

    diff --git a/docs/dli/umn/dli_01_0451.html b/docs/dli/umn/dli_01_0451.html index f0caa1f18..ef9bda21d 100644 --- a/docs/dli/umn/dli_01_0451.html +++ b/docs/dli/umn/dli_01_0451.html @@ -262,7 +262,7 @@ } -

    Example Custom Policies